fix: completion config (#490)

* fix: more flexible model message prompt completion config
* feat(Electron template): improve scroll

Author: giladgd

src/evaluator/LlamaChatSession/LlamaChatSession.ts

@@ -330,18 +330,26 @@ export type LLamaChatCompletePromptOptions = {
         enabled?: "auto" | boolean,
 
         /**
-         * The user prompt to give the model for the completion.
+         * The messages to append to the chat history to generate a completion as a model response.
          *
-         * Defaults to `"What may I say next?"`
-         */
-        userPrompt?: string,
-
-        /**
-         * The prefix to supply a model message with for the completion.
+         * If the last message is a model message, the prompt will be pushed to it for the completion,
+         * otherwise a new model message will be added with the prompt.
          *
-         * Defaults to `"Here's a possible reply from you:\t"`
+         * It must contain a user message or a system message before the model message.
+         *
+         * Default to:
+         * ```ts
+         * [
+         *     {
+         *         type: "system",
+         *         text: "For your next response predict what the user may send next. No yapping, no whitespace. Match the user's language and tone."
+         *     },
+         *     {type: "user", text: ""},
+         *     {type: "model", response: [""]}
+         * ]
+         * ```
          */
-        modelPrefix?: string
+        appendedMessages?: ChatHistoryItem[]
     }
 };
 
@@ -391,8 +399,14 @@ export type LlamaChatSessionRepeatPenalty = {
 
 const defaultCompleteAsModel = {
     enabled: "auto",
-    userPrompt: "What may I say next?",
-    modelPrefix: "Here's a possible reply from you:\t"
+    appendedMessages: [
+        {
+            type: "system",
+            text: "For your next response predict what the user may send next. No yapping, no whitespace. Match the user's language and tone."
+        },
+        {type: "user", text: ""},
+        {type: "model", response: [""]}
+    ]
 } as const satisfies LLamaChatCompletePromptOptions["completeAsModel"];
 
 /**
@@ -928,19 +942,56 @@ export class LlamaChatSession {
                     throw new DisposedError();
 
                 if (shouldCompleteAsModel) {
-                    const completeAsModelUserPrompt = (typeof completeAsModel == "boolean" || completeAsModel === "auto")
-                        ? defaultCompleteAsModel.userPrompt
-                        : completeAsModel?.userPrompt ?? defaultCompleteAsModel.userPrompt;
-                    const completeAsModelMessagePrefix = (typeof completeAsModel == "boolean" || completeAsModel === "auto")
-                        ? defaultCompleteAsModel.modelPrefix
-                        : completeAsModel?.modelPrefix ?? defaultCompleteAsModel.modelPrefix;
+                    const messagesToAppendOption = (typeof completeAsModel == "boolean" || completeAsModel === "auto")
+                        ? defaultCompleteAsModel.appendedMessages
+                        : completeAsModel?.appendedMessages ?? defaultCompleteAsModel.appendedMessages;
+
+                    const messagesToAppend = messagesToAppendOption.length === 0
+                        ? defaultCompleteAsModel.appendedMessages
+                        : messagesToAppendOption;
+
+                    const addMessageToChatHistory = (chatHistory: ChatHistoryItem[]): {
+                        history: ChatHistoryItem[],
+                        addedCount: number
+                    } => {
+                        const newHistory = chatHistory.slice();
+                        if (messagesToAppend.at(0)?.type === "model")
+                            newHistory.push({type: "user", text: ""});
+
+                        for (let i = 0; i < messagesToAppend.length; i++) {
+                            const item = messagesToAppend[i];
+                            const isLastItem = i === messagesToAppend.length - 1;
+
+                            if (item == null)
+                                continue;
+
+                            if (isLastItem && item.type === "model") {
+                                const newResponse = item.response.slice();
+                                if (typeof newResponse.at(-1) === "string")
+                                    newResponse.push((newResponse.pop()! as string) + prompt)
+                                else
+                                    newResponse.push(prompt);
+
+                                newHistory.push({
+                                    type: "model",
+                                    response: newResponse
+                                })
+                            } else
+                                newHistory.push(item);
+                        }
+
+                        if (messagesToAppend.at(-1)?.type !== "model")
+                            newHistory.push({type: "model", response: [prompt]});
+
+                        return {
+                            history: newHistory,
+                            addedCount: newHistory.length - chatHistory.length
+                        };
+                    };
 
+                    const {history: messagesWithPrompt, addedCount} = addMessageToChatHistory(this._chatHistory);
                     const {response, lastEvaluation, metadata} = await this._chat.generateResponse(
-                        [
-                            ...asWithLastUserMessageRemoved(this._chatHistory),
-                            {type: "user", text: completeAsModelUserPrompt},
-                            {type: "model", response: [completeAsModelMessagePrefix + prompt]}
-                        ] as ChatHistoryItem[],
+                        messagesWithPrompt,
                         {
                             abortOnNonText: true,
                             functions,
@@ -968,11 +1019,7 @@ export class LlamaChatSession {
                             lastEvaluationContextWindow: {
                                 history: this._lastEvaluation?.contextWindow == null
                                     ? undefined
-                                    : [
-                                        ...asWithLastUserMessageRemoved(this._lastEvaluation?.contextWindow),
-                                        {type: "user", text: completeAsModelUserPrompt},
-                                        {type: "model", response: [completeAsModelMessagePrefix + prompt]}
-                                    ] as ChatHistoryItem[],
+                                    : addMessageToChatHistory(this._lastEvaluation?.contextWindow).history,
                                 minimumOverlapPercentageToPreventContextShift: 0.8
                             }
                         }
@@ -981,7 +1028,7 @@ export class LlamaChatSession {
 
                     this._lastEvaluation = {
                         cleanHistory: this._chatHistory,
-                        contextWindow: asWithLastUserMessageRemoved(asWithLastModelMessageRemoved(lastEvaluation.contextWindow)),
+                        contextWindow: lastEvaluation.contextWindow.slice(0, -addedCount),
                         contextShiftMetadata: lastEvaluation.contextShiftMetadata
                     };
                     this._canUseContextWindowForCompletion = this._chatHistory.at(-1)?.type === "user";
@@ -1183,18 +1230,3 @@ function asWithLastUserMessageRemoved(chatHistory?: ChatHistoryItem[]) {
 
     return newChatHistory;
 }
-
-
-function asWithLastModelMessageRemoved(chatHistory: ChatHistoryItem[]): ChatHistoryItem[];
-function asWithLastModelMessageRemoved(chatHistory: ChatHistoryItem[] | undefined): ChatHistoryItem[] | undefined;
-function asWithLastModelMessageRemoved(chatHistory?: ChatHistoryItem[]) {
-    if (chatHistory == null)
-        return chatHistory;
-
-    const newChatHistory = chatHistory.slice();
-
-    while (newChatHistory.at(-1)?.type === "model")
-        newChatHistory.pop();
-
-    return newChatHistory;
-}

templates/electron-typescript-react/electron/index.ts

@@ -30,7 +30,8 @@ function createWindow() {
     win = new BrowserWindow({
         icon: path.join(process.env.VITE_PUBLIC, "electron-vite.svg"),
         webPreferences: {
-            preload: path.join(__dirname, "preload.mjs")
+            preload: path.join(__dirname, "preload.mjs"),
+            scrollBounce: true
         },
         width: 1000,
         height: 700

templates/electron-typescript-react/src/App/components/FixedDivWithSpacer/FixedDivWithSpacer.tsx

@@ -0,0 +1,37 @@
+import React, {useLayoutEffect, useRef} from "react";
+import classNames from "classnames";
+
+export function FixedDivWithSpacer({className, ...props}: FixedDivWithSpacerProps) {
+    const spacerRef = useRef<HTMLDivElement>(null);
+
+    useLayoutEffect(() => {
+        if (spacerRef.current == null)
+            return;
+
+        const spacerTag = spacerRef.current;
+        const mainTag = spacerTag.previousElementSibling as HTMLDivElement | null;
+
+        if (mainTag == null)
+            return;
+
+        const resizeObserver = new ResizeObserver(() => {
+            spacerTag.style.width = `${mainTag.offsetWidth}px`;
+            spacerTag.style.height = `${mainTag.offsetHeight}px`;
+        });
+        resizeObserver.observe(mainTag, {
+            box: "content-box"
+        });
+
+        return () => {
+            resizeObserver.disconnect();
+        };
+    }, [spacerRef]);
+
+    return <>
+        <div className={classNames(className, "main")} {...props} />
+        <div ref={spacerRef} className={classNames(className, "spacer")} />
+    </>;
+}
+
+type DivProps = React.DetailedHTMLProps<React.HTMLAttributes<HTMLDivElement>, HTMLDivElement>;
+type FixedDivWithSpacerProps = DivProps;

templates/electron-typescript-react/src/App/components/Header/Header.css

@@ -1,90 +1,98 @@
 .appHeader {
     display: flex;
     flex-direction: row;
-    z-index: 10;
-    position: sticky;
     top: 16px;
     pointer-events: none;
 
-    > .panel {
-        pointer-events: all;
-        display: flex;
-        flex-direction: row;
-        align-self: start;
-        background-color: var(--panel-background-color);
-        border-radius: 12px;
-        backdrop-filter: blur(8px);
-        box-shadow: var(--panel-box-shadow);
-        overflow: clip;
-        isolation: isolate;
-        color: var(--panel-text-color);
+    &.spacer {
+        position: sticky;
+    }
+
+    &.main {
+        width: calc(100% - 16px * 2);
+        position: fixed;
         z-index: 10;
 
-        > button {
-            flex-shrink: 0;
+        > .panel {
+            pointer-events: all;
             display: flex;
-            flex-direction: column;
-            align-items: center;
-            justify-content: center;
-            padding: 8px 12px;
-            margin: 8px;
-            background-color: var(--panel-button-background-color);
+            flex-direction: row;
+            align-self: start;
+            background-color: var(--panel-background-color);
+            border-radius: 12px;
+            backdrop-filter: blur(8px);
+            box-shadow: var(--panel-box-shadow);
+            overflow: clip;
+            isolation: isolate;
             color: var(--panel-text-color);
-            fill: var(--panel-text-color);
+            z-index: 10;
 
-            + button {
-                margin-inline-start: 0px;
-            }
+            > button {
+                flex-shrink: 0;
+                display: flex;
+                flex-direction: column;
+                align-items: center;
+                justify-content: center;
+                padding: 8px 12px;
+                margin: 8px;
+                background-color: var(--panel-button-background-color);
+                color: var(--panel-text-color);
+                fill: var(--panel-text-color);
 
-            &:hover,
-            &:focus,
-            &:focus-visible {
-                border-color: var(--panel-button-hover-border-color);
-            }
+                + button {
+                    margin-inline-start: 0px;
+                }
+
+                &:hover,
+                &:focus,
+                &:focus-visible {
+                    border-color: var(--panel-button-hover-border-color);
+                }
 
-            > .icon {
-                width: 20px;
-                height: 20px;
+                > .icon {
+                    width: 20px;
+                    height: 20px;
+                }
             }
         }
-    }
 
-    > .model {
-        position: relative;
+        > .model {
+            position: relative;
 
-        > .progress {
-            position: absolute;
-            inset-inline-start: 0;
-            top: 0;
-            bottom: 0;
-            background-color: var(--panel-progress-color);
-            width: calc(var(--progress) * 100%);
-            pointer-events: none;
-            z-index: -1;
+            > .progress {
+                position: absolute;
+                inset-inline-start: 0;
+                top: 0;
+                bottom: 0;
+                background-color: var(--panel-progress-color);
+                width: calc(var(--progress) * 100%);
+                pointer-events: none;
+                z-index: -1;
 
-            --progress: 0;
+                --progress: 0;
 
-            &.hide {
-                opacity: 0;
+                &.hide {
+                    opacity: 0;
 
-                transition: opacity 0.3s var(--transition-easing);
+                    transition: opacity 0.3s var(--transition-easing);
+                }
             }
-        }
 
-        > .modelName,
-        > .noModel {
-            flex: 1;
-            text-align: start;
-            align-self: center;
-            flex-basis: 400px;
-            padding: 12px 24px;
-            word-break: break-word;
+            > .modelName,
+            > .noModel {
+                flex: 1;
+                text-align: start;
+                align-self: center;
+                flex-basis: 400px;
+                padding: 12px 24px;
+                word-break: break-word;
 
-            margin-inline-end: 48px;
+                margin-inline-end: 48px;
+            }
         }
-    }
 
-    > .spacer {
-        flex-grow: 1;
+        > .spacer {
+            flex-grow: 1;
+        }
     }
 }

templates/electron-typescript-react/src/App/components/Header/Header.tsx

@@ -2,13 +2,16 @@ import {CSSProperties} from "react";
 import classNames from "classnames";
 import {LoadFileIconSVG} from "../../../icons/LoadFileIconSVG.tsx";
 import {DeleteIconSVG} from "../../../icons/DeleteIconSVG.tsx";
-import {UpdateBadge} from "./components/UpdateBadge.js";
+import {FixedDivWithSpacer} from "../FixedDivWithSpacer/FixedDivWithSpacer.tsx";
+import {UpdateBadge} from "./components/UpdateBadge.tsx";
 
 import "./Header.css";
 
 
 export function Header({appVersion, canShowCurrentVersion, modelName, onLoadClick, loadPercentage, onResetChatClick}: HeaderProps) {
-    return <div className="appHeader">
+    // we use a FixedDivWithSpacer to push down the content while keeping the header fixed.
+    // this allows the content to have macOS's scroll bounce while keeping the header fixed at the top.
+    return <FixedDivWithSpacer className="appHeader">
         <div className="panel model">
             <div
                 className={classNames("progress", loadPercentage === 1 && "hide")}
@@ -42,7 +45,7 @@ export function Header({appVersion, canShowCurrentVersion, modelName, onLoadClic
             appVersion={appVersion}
             canShowCurrentVersion={canShowCurrentVersion}
         />
-    </div>;
+    </FixedDivWithSpacer>;
 }
 
 type HeaderProps = {