Update the timeout value on all stream-timeout providers: (#4412)

- OpenRouter
- Novita
- CometAPI
updated to 3,000ms default with 500ms min

Author: timothycarambat

frontend/src/components/LLMSelection/CometApiLLMOptions/index.jsx

@@ -59,7 +59,7 @@ function AdvancedControls({ settings }) {
             name="CometApiLLMTimeout"
             className="border-none bg-theme-settings-input-bg text-theme-text-primary placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="Timeout value between token responses to auto-timeout the stream"
-            defaultValue={settings?.CometApiLLMTimeout ?? 500}
+            defaultValue={settings?.CometApiLLMTimeout ?? 3_000}
             autoComplete="off"
             onScroll={(e) => e.target.blur()}
             min={500}

frontend/src/components/LLMSelection/NovitaLLMOptions/index.jsx

@@ -59,7 +59,7 @@ function AdvancedControls({ settings }) {
             name="NovitaLLMTimeout"
             className="border-none bg-theme-settings-input-bg text-theme-text-primary placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="Timeout value between token responses to auto-timeout the stream"
-            defaultValue={settings?.NovitaLLMTimeout ?? 500}
+            defaultValue={settings?.NovitaLLMTimeout ?? 3_000}
             autoComplete="off"
             onScroll={(e) => e.target.blur()}
             min={500}

frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx

@@ -57,7 +57,7 @@ function AdvancedControls({ settings }) {
             name="OpenRouterTimeout"
             className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="Timeout value between token responses to auto-timeout the stream"
-            defaultValue={settings?.OpenRouterTimeout ?? 500}
+            defaultValue={settings?.OpenRouterTimeout ?? 3_000}
             autoComplete="off"
             onScroll={(e) => e.target.blur()}
             min={500}

server/utils/AiProviders/cometapi/index.js

@@ -19,6 +19,7 @@ const cacheFolder = path.resolve(
 );
 
 class CometApiLLM {
+  defaultTimeout = 3_000;
   constructor(embedder = null, modelPreference = null) {
     if (!process.env.COMETAPI_LLM_API_KEY)
       throw new Error("No CometAPI API key was set.");
@@ -61,10 +62,14 @@ class CometApiLLM {
    * CometAPI has various models that never return `finish_reasons` and thus leave the stream open
    * which causes issues in subsequent messages. This timeout value forces us to close the stream after
    * x milliseconds. This is a configurable value via the COMETAPI_LLM_TIMEOUT_MS value
-   * @returns {number} The timeout value in milliseconds (default: 500)
+   * @returns {number} The timeout value in milliseconds (default: 3_000)
    */
   #parseTimeout() {
-    if (isNaN(Number(process.env.COMETAPI_LLM_TIMEOUT_MS))) return 500;
+    this.log(
+      `CometAPI timeout is set to ${process.env.COMETAPI_LLM_TIMEOUT_MS ?? this.defaultTimeout}ms`
+    );
+    if (isNaN(Number(process.env.COMETAPI_LLM_TIMEOUT_MS)))
+      return this.defaultTimeout;
     const setValue = Number(process.env.COMETAPI_LLM_TIMEOUT_MS);
     if (setValue < 500) return 500;
     return setValue;

server/utils/AiProviders/novita/index.js

@@ -18,6 +18,8 @@ const cacheFolder = path.resolve(
 );
 
 class NovitaLLM {
+  defaultTimeout = 3_000;
+
   constructor(embedder = null, modelPreference = null) {
     if (!process.env.NOVITA_LLM_API_KEY)
       throw new Error("No Novita API key was set.");
@@ -62,12 +64,16 @@ class NovitaLLM {
    * Novita has various models that never return `finish_reasons` and thus leave the stream open
    * which causes issues in subsequent messages. This timeout value forces us to close the stream after
    * x milliseconds. This is a configurable value via the NOVITA_LLM_TIMEOUT_MS value
-   * @returns {number} The timeout value in milliseconds (default: 500)
+   * @returns {number} The timeout value in milliseconds (default: 3_000)
    */
   #parseTimeout() {
-    if (isNaN(Number(process.env.NOVITA_LLM_TIMEOUT_MS))) return 500;
+    this.log(
+      `Novita timeout is set to ${process.env.NOVITA_LLM_TIMEOUT_MS ?? this.defaultTimeout}ms`
+    );
+    if (isNaN(Number(process.env.NOVITA_LLM_TIMEOUT_MS)))
+      return this.defaultTimeout;
     const setValue = Number(process.env.NOVITA_LLM_TIMEOUT_MS);
-    if (setValue < 500) return 500;
+    if (setValue < 500) return 500; // 500ms is the minimum timeout
     return setValue;
   }
 
@@ -318,7 +324,7 @@ class NovitaLLM {
             });
           }
 
-          if (message.finish_reason !== null) {
+          if (message?.finish_reason !== null) {
             writeResponseChunk(response, {
               uuid,
               sources,

server/utils/AiProviders/openRouter/index.js

@@ -18,6 +18,16 @@ const cacheFolder = path.resolve(
 );
 
 class OpenRouterLLM {
+  /**
+   * Some openrouter models never send a finish_reason and thus leave the stream open in the UI.
+   * However, because OR is a middleware it can also wait an inordinately long time between chunks so we need
+   * to ensure that we dont accidentally close the stream too early. If the time between chunks is greater than this timeout
+   * we will close the stream and assume it to be complete. This is common for free models or slow providers they can
+   * possibly delegate to during invocation.
+   * @type {number}
+   */
+  defaultTimeout = 3_000;
+
   constructor(embedder = null, modelPreference = null) {
     if (!process.env.OPENROUTER_API_KEY)
       throw new Error("No OpenRouter API key was set.");
@@ -85,15 +95,16 @@ class OpenRouterLLM {
    * OpenRouter has various models that never return `finish_reasons` and thus leave the stream open
    * which causes issues in subsequent messages. This timeout value forces us to close the stream after
    * x milliseconds. This is a configurable value via the OPENROUTER_TIMEOUT_MS value
-   * @returns {number} The timeout value in milliseconds (default: 500)
+   * @returns {number} The timeout value in milliseconds (default: 3_000)
    */
   #parseTimeout() {
     this.log(
-      `OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? 500}ms`
+      `OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? this.defaultTimeout}ms`
     );
-    if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS))) return 500;
+    if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS)))
+      return this.defaultTimeout;
     const setValue = Number(process.env.OPENROUTER_TIMEOUT_MS);
-    if (setValue < 500) return 500;
+    if (setValue < 500) return 500; // 500ms is the minimum timeout
     return setValue;
   }