docs / type fixes

Author: YousefED

docs/pages/docs/ai/reference.mdx

@@ -21,8 +21,8 @@ type AIExtensionOptions = {
   stream?: boolean;
   /**
    * The default data format to use for LLM calls
-   * "html" is recommended, the other formats are experimental
-   * @default html
+   * html format is recommended, the other formats are experimental
+   * @default llmFormats.html
    */
   dataFormat?: "html" | "json" | "markdown";
   /**
@@ -56,30 +56,46 @@ class AIExtension {
   /**
    * Execute a call to an LLM and apply the result to the editor
    */
-  callLLM(opts: CallSpecificCallLLMOptions): Promise<void>;
+  callLLM(
+    opts: MakeOptional<LLMRequestOptions, "model">,
+  ): Promise<LLMResponse | undefined>;
   /**
    * Returns a read-only zustand store with the state of the AI Menu
    */
   get store(): ReadonlyStoreApi<{
     aiMenuState:
-      | {
+      | ({
+          /**
+           * The ID of the block that the AI menu is open at
+           * this changes as the AI is making changes to the document
+           */
           blockId: string;
-          status:
-            | "user-input"
-            | "thinking"
-            | "ai-writing"
-            | "error"
-            | "user-reviewing";
-        }
+        } & (
+          | {
+              status: "error";
+              error: any;
+            }
+          | {
+              status:
+                | "user-input"
+                | "thinking"
+                | "ai-writing"
+                | "user-reviewing";
+            }
+        ))
       | "closed";
+    /**
+     * The previous response from the LLM, used for multi-step LLM calls
+     */
+    llmResponse?: LLMResponse;
   }>;
   /**
    * Returns a zustand store with the global configuration of the AI Extension,
    * these options are used as default across all LLM calls when calling {@link callLLM}
    */
   readonly options: StoreApi<{
     model: LanguageModel;
-    dataFormat: "html" | "json" | "markdown";
+    dataFormat: LLMFormat;
     stream: boolean;
     promptBuilder?: PromptBuilder;
   }>;
@@ -100,6 +116,12 @@ class AIExtension {
    * Reject the changes made by the LLM
    */
   rejectChanges(): void;
+  /**
+   * Retry the previous LLM call.
+   *
+   * Only valid if the current status is "error"
+   */
+  retry(): Promise<void>;
   /**
    * Update the status of a call to an LLM
    *
@@ -112,24 +134,44 @@ class AIExtension {
       | "user-input"
       | "thinking"
       | "ai-writing"
-      | "error"
-      | "user-reviewing",
+      | "user-reviewing"
+      | {
+          status: "error";
+          error: any;
+        },
   ): void;
 }
 ```
 
-## `callLLM`
+### `LLMRequestOptions`
+
+Requests an LLM are made by calling `callLLM` on the `AIExtension` object.
 
 ```typescript
-type CallLLMOptions = {
+type LLMRequestOptions = {
   /**
    * The language model to use for the LLM call (AI SDK)
+   *
+   * (when invoking `callLLM` via the `AIExtension` this will default to the
+   * model set in the `AIExtension` options)
    */
   model: LanguageModelV1;
   /**
    * The user prompt to use for the LLM call
    */
   userPrompt: string;
+  /**
+   * Previous response from the LLM, used for multi-step LLM calls
+   *
+   * (populated automatically when invoking `callLLM` via the `AIExtension` class)
+   */
+  previousResponse?: LLMResponse;
+  /**
+   * The default data format to use for LLM calls
+   * "html" is recommended, the other formats are experimental
+   * @default html format (`llm.html`)
+   */
+  dataFormat?: LLMFormat;
   /**
    * The `PromptBuilder` to use for the LLM call
    *
@@ -199,14 +241,129 @@ type CallLLMOptions = {
    */
   withDelays?: boolean;
   /**
-   * Additional options to pass to the `generateObject` function
+   * Additional options to pass to the AI SDK `generateObject` function
    * (only used when `stream` is `false`)
    */
   _generateObjectOptions?: Partial<Parameters<typeof generateObject<any>>[0]>;
   /**
-   * Additional options to pass to the `streamObject` function
+   * Additional options to pass to the AI SDK `streamObject` function
    * (only used when `stream` is `true`)
    */
   _streamObjectOptions?: Partial<Parameters<typeof streamObject<any>>[0]>;
 };
 ```
+
+## (advanced) `doLLMRequest`
+
+The `CallLLM` function automatically passes the default options set in the `AIExtension` to the LLM request.
+It also handles the LLM response and the AI menu state
+
+For advanced use cases, you can also directly use the `doLLMRequest` to issue an LLM request.
+In this case, you will need to manually handle the response.
+
+```typescript
+/**
+ * Execute an LLM call
+ *
+ * @param editor - The BlockNoteEditor the LLM should operate on
+ * @param opts - The options for the LLM call (@link {CallLLMOptions})
+ * @returns A `LLMResponse` object containing the LLM response which can be applied to the editor
+ */
+function doLLMRequest(
+  editor: BlockNoteEditor<any, any, any>,
+  opts: LLMRequestOptions,
+): Promise<LLMResponse>;
+```
+
+Call `execute` on the `LLMResponse` object to apply the changes to the editor.
+
+## PromptBuilder (advanced)
+
+The `PromptBuilder` is a function that takes a BlockNoteEditor and details about the user's prompt
+and turns it into an array of CoreMessage objects (AI SDK) to be passed to the LLM.
+
+Providing a custom `PromptBuilder` allows fine-grained control over the instructions sent to the LLM.
+To implement a custom `PromptBuilder`, you can use the `promptHelpers` provided by the LLM format.
+We recommend looking at the [default PromptBuilder](https://github.com/TypeCellOS/BlockNote/blob/main/packages/xl-ai/src/api/formats/html-blocks/defaultHTMLPromptBuilder.ts) implementations as a starting point to implement your own.
+
+```typescript
+/**
+ * The input passed to a PromptBuilder
+ */
+type PromptBuilderInput = {
+  /**
+   * The user's prompt
+   */
+  userPrompt: string;
+  /**
+   * The selection of the editor which the LLM should operate on
+   */
+  selectedBlocks?: Block<any, any, any>[];
+  /**
+   * The ids of blocks that should be excluded from the prompt
+   * (e.g.: if `deleteEmptyCursorBlock` is true in the LLMRequest,
+   * this will be the id of the block that should be ignored)
+   */
+  excludeBlockIds?: string[];
+  /**
+   * When following a multi-step conversation, or repairing a previous error,
+   * the previous messages that have been sent to the LLM
+   */
+  previousMessages?: Array<CoreMessage>;
+};
+
+/**
+ * A PromptBuilder is a function that takes a BlockNoteEditor and details about the user's promot
+ * and turns it into an array of CoreMessage (AI SDK) to be passed to the LLM.
+ */
+type PromptBuilder = (
+  editor: BlockNoteEditor<any, any, any>,
+  opts: PromptBuilderInput,
+) => Promise<Array<CoreMessage>>;
+```
+
+## Formats
+
+When a LLM is called, the LLM needs to interpret the document, and invoke operations to modify the document.
+Different models might be able to understand different data formats better.
+By default, BlockNote and LLM models interoperate using a HTML based format. We also provide experimental JSON and Markdown based formats.
+
+```typescript
+type LLMFormat = {
+  /**
+   * Function to get th format specific stream tools that the LLM can choose to invoke
+   */
+  getStreamTools: (
+    editor: BlockNoteEditor<any, any, any>,
+    withDelays: boolean,
+    defaultStreamTools?: {
+      add?: boolean;
+      update?: boolean;
+      delete?: boolean;
+    },
+    selectionInfo?: {
+      from: number;
+      to: number;
+    },
+    onBlockUpdate?: (blockId: string) => void,
+  ) => StreamTool<any>[];
+  /**
+   * The default PromptBuilder that determines how a userPrompt is converted to an array of
+   * LLM Messages (CoreMessage[])
+   */
+  defaultPromptBuilder: PromptBuilder;
+  /**
+   * Helper functions which can be used when implementing a custom PromptBuilder.
+   * The signature depends on the specific format
+   */
+  promptHelpers: any;
+};
+
+// The default LLMFormats are exported under `llmFormats`:
+
+export const llmFormats = {
+  _experimental_json,
+  _experimental_markdown,
+  html,
+};
+```

packages/xl-ai/src/AIExtension.ts

@@ -26,16 +26,18 @@ type ReadonlyStoreApi<T> = Pick<
 >;
 
 type AIPluginState = {
-  /**
-   * zustand design considerations:
-   * - moved this to a nested object to have better typescript typing
-   * - if we'd do this without a nested object, then we could easily set "wrong" values,
-   *   because "setState" takes a partial object (unless the second parameter "replace" = true),
-   *   and thus we'd lose typescript's typing help
-   *
-   */
+  // zustand design considerations:
+  //  - moved this to a nested object to have better typescript typing
+  //  - if we'd do this without a nested object, then we could easily set "wrong" values,
+  //    because "setState" takes a partial object (unless the second parameter "replace" = true),
+  //    and thus we'd lose typescript's typing help
+
   aiMenuState:
     | ({
+        /**
+         * The ID of the block that the AI menu is open at
+         * this changes as the AI is making changes to the document
+         */
         blockId: string;
       } & (
         | {
@@ -64,8 +66,8 @@ type GlobalLLMRequestOptions = {
   model: LanguageModel;
   /**
    * The default data format to use for LLM calls
-   * "html" is recommended, the other formats are experimental
-   * @default html
+   * html format is recommended, the other formats are experimental
+   * @default llmFormats.html
    */
   dataFormat?: LLMFormat;
   /**
@@ -80,12 +82,10 @@ type GlobalLLMRequestOptions = {
   promptBuilder?: PromptBuilder;
 };
 
-type CallSpecificLLMRequestOptions = MakeOptional<LLMRequestOptions, "model">;
-
 const PLUGIN_KEY = new PluginKey(`blocknote-ai-plugin`);
 
 export class AIExtension extends BlockNoteExtension {
-  private previousRequestOptions: CallSpecificLLMRequestOptions | undefined;
+  private previousRequestOptions: LLMRequestOptions | undefined;
 
   public static name(): string {
     return "ai";
@@ -297,9 +297,9 @@ export class AIExtension extends BlockNoteExtension {
   /**
    * Execute a call to an LLM and apply the result to the editor
    */
-  public async callLLM(opts: CallSpecificLLMRequestOptions) {
+  public async callLLM(opts: MakeOptional<LLMRequestOptions, "model">) {
     this.setAIResponseStatus("thinking");
-
+    let ret: LLMResponse | undefined;
     try {
       const requestOptions = {
         ...this.options.getState(),
@@ -308,10 +308,11 @@ export class AIExtension extends BlockNoteExtension {
       };
       this.previousRequestOptions = requestOptions;
 
-      const ret = await doLLMRequest(this.editor, {
+      ret = await doLLMRequest(this.editor, {
         ...requestOptions,
         onStart: () => {
           this.setAIResponseStatus("ai-writing");
+          opts.onStart?.();
         },
         onBlockUpdate: (blockId: string) => {
           // NOTE: does this setState with an anon object trigger unnecessary re-renders?
@@ -321,6 +322,7 @@ export class AIExtension extends BlockNoteExtension {
               status: "ai-writing",
             },
           });
+          opts.onBlockUpdate?.(blockId);
         },
       });
 
@@ -339,6 +341,7 @@ export class AIExtension extends BlockNoteExtension {
       // eslint-disable-next-line no-console
       console.warn("Error calling LLM", e);
     }
+    return ret;
   }
 }
 

packages/xl-ai/src/api/LLMRequest.ts

@@ -13,6 +13,9 @@ import { LLMFormat } from "./index.js";
 export type LLMRequestOptions = {
   /**
    * The language model to use for the LLM call (AI SDK)
+   *
+   * (when invoking `callLLM` via the `AIExtension` this will default to the
+   * model set in the `AIExtension` options)
    */
   model: LanguageModelV1;
   /**
@@ -21,6 +24,8 @@ export type LLMRequestOptions = {
   userPrompt: string;
   /**
    * Previous response from the LLM, used for multi-step LLM calls
+   *
+   * (populated automatically when invoking `callLLM` via the `AIExtension` class)
    */
   previousResponse?: LLMResponse;
   /**
@@ -98,19 +103,19 @@ export type LLMRequestOptions = {
    */
   withDelays?: boolean;
   /**
-   * Additional options to pass to the `generateObject` function
+   * Additional options to pass to the AI SDK `generateObject` function
    * (only used when `stream` is `false`)
    */
   _generateObjectOptions?: Partial<Parameters<typeof generateObject<any>>[0]>;
   /**
-   * Additional options to pass to the `streamObject` function
+   * Additional options to pass to the AI SDK `streamObject` function
    * (only used when `stream` is `true`)
    */
   _streamObjectOptions?: Partial<Parameters<typeof streamObject<any>>[0]>;
 };
 
 /**
- * Execute an LLM call and apply the result to the editor
+ * Execute an LLM call
  *
  * @param editor - The BlockNoteEditor the LLM should operate on
  * @param opts - The options for the LLM call (@link {CallLLMOptions})