llm: disable gpt 3.5 and 3.5 16k and fix+test getValidLanguageModelName

Author: haraldschilly

src/packages/util/db-schema/llm-utils.test.ts

@@ -1,9 +1,14 @@
 import {
   DEFAULT_LLM_PRIORITY,
+  DEFAULT_MODEL,
+  getValidLanguageModelName,
+  isCoreLanguageModel,
   isFreeModel,
   LANGUAGE_MODEL_SERVICES,
   LANGUAGE_MODELS,
+  LanguageService,
   LLM_COST,
+  LLMServicesAvailable,
   model2vendor,
   OLLAMA_PREFIX,
   SERVICES,
@@ -83,4 +88,64 @@ describe("llm", () => {
       expect(prio.includes(v)).toBe(true);
     }
   });
+
+  test("getting valid language model", () => {
+    const selectable_llms = [...USER_SELECTABLE_LANGUAGE_MODELS];
+    const notAvailable = selectable_llms.pop();
+
+    function getModel(model: LanguageService, disabled?: LanguageService) {
+      const allEnabled = LANGUAGE_MODEL_SERVICES.reduce((acc, svc) => {
+        acc[svc] = disabled !== svc;
+        return acc;
+      }, {}) as LLMServicesAvailable;
+      return getValidLanguageModelName({
+        model,
+        filter: allEnabled,
+        ollama: ["phi3"],
+        custom_openai: ["bar"],
+        selectable_llms,
+      });
+    }
+
+    // meaningless name
+    expect(getModel("foobar")).toEqual(DEFAULT_MODEL);
+    expect(getModel("baz-delta99")).toEqual(DEFAULT_MODEL);
+    // gpt 3.5 is disabled
+    expect(getModel("gpt-3.5-turbo")).toEqual(DEFAULT_MODEL);
+    // not available
+    expect(
+      typeof notAvailable === "string" && isCoreLanguageModel(notAvailable),
+    ).toBe(true);
+    if (typeof notAvailable === "string") {
+      expect(getModel(notAvailable)).toEqual(DEFAULT_MODEL);
+    }
+    // not disabled
+    expect(getModel("mistral-large-latest")).toEqual("mistral-large-latest");
+    expect(getModel("gpt-4")).toEqual("gpt-4");
+    expect(getModel(DEFAULT_MODEL)).toEqual(DEFAULT_MODEL);
+    expect(getModel("mistral-medium-latest")).toEqual(DEFAULT_MODEL);
+    expect(getModel("mistral-large-latest")).toEqual("mistral-large-latest");
+    expect(getModel("claude-3-haiku-8k")).toEqual("claude-3-haiku-8k");
+    // anthropic service disabled
+    expect(getModel("claude-3-haiku-8k", "anthropic")).toEqual(DEFAULT_MODEL);
+    // ollama
+    expect(getModel("ollama-foo")).toEqual(DEFAULT_MODEL);
+    expect(getModel("ollama-phi3")).toEqual("ollama-phi3");
+    // openai api
+    expect(getModel("custom_openai-foo")).toEqual(DEFAULT_MODEL);
+    expect(getModel("custom_openai-bar")).toEqual("custom_openai-bar");
+    // user models: there are no further checks
+    expect(getModel("user-custom_openai-foo")).toEqual(
+      "user-custom_openai-foo",
+    );
+    expect(getModel("user-openai-gpt-3.5-turbo")).toEqual(
+      "user-openai-gpt-3.5-turbo",
+    );
+    // it's ok to use a model if disabled by the admin, since it's their key
+    expect(getModel("user-anthropic-claude-3-haiku-8k", "anthropic")).toEqual(
+      "user-anthropic-claude-3-haiku-8k",
+    );
+    // meaningless user service
+    expect(getModel("user-baz-delta99")).toEqual(DEFAULT_MODEL);
+  });
 });

src/packages/util/db-schema/llm-utils.ts

@@ -214,12 +214,7 @@ export const USER_SELECTABLE_LLMS_BY_VENDOR: {
   [vendor in LLMServiceName]: Readonly<LanguageModelCore[]>;
 } = {
   openai: MODELS_OPENAI.filter(
-    (m) =>
-      m === "gpt-3.5-turbo" ||
-      m === "gpt-3.5-turbo-16k" ||
-      m === "gpt-4" ||
-      m === "gpt-4-turbo-preview-8k" ||
-      m === "gpt-4o-8k",
+    (m) => m === "gpt-4" || m === "gpt-4-turbo-preview-8k" || m === "gpt-4o-8k",
   ),
   google: GOOGLE_MODELS.filter(
     (m) =>
@@ -367,20 +362,28 @@ export function getValidLanguageModelName({
 }: ValidLanguageModelNameProps): LanguageModel {
   if (typeof model === "string" && isValidModel(model)) {
     try {
-      const v = model2vendor(model).name;
-      if (filter[v]) {
+      if (isCoreLanguageModel(model)) {
+        const v = model2vendor(model).name;
+        if (filter[v] && selectable_llms.includes(model)) {
+          return model;
+        }
+      }
+
+      if (isOllamaLLM(model) && ollama.includes(fromOllamaModel(model))) {
+        return model;
+      }
+
+      if (
+        isCustomOpenAI(model) &&
+        custom_openai.includes(fromCustomOpenAIModel(model))
+      ) {
+        return model;
+      }
+
+      if (isUserDefinedModel(model)) {
         return model;
       }
     } catch {}
-    if (isOllamaLLM(model) && ollama.includes(fromOllamaModel(model))) {
-      return model;
-    }
-    if (
-      isCustomOpenAI(model) &&
-      custom_openai.includes(fromCustomOpenAIModel(model))
-    ) {
-      return model;
-    }
   }
 
   for (const free of [true, false]) {