lib: add threepio to start using LLM for rendering contents progressively (#75)

Author: EndlessJour9527

docs/development-zh.md

@@ -138,6 +138,27 @@ $ adb shell setprop jsar.renderer.target_app_fps 60
 
 默认为 45 帧。
 
+**使用 LLM**
+
+```sh
+## 配置使用 LLM API 的模型 例如 `qwen-plus-latest`
+$ adb shell setprop jsar.threepio.api.modelid your-llm-modelid
+## 配置 LLM API 服务的组织/平台 例如 `qwen`
+$ adb shell setprop jsar.threepio.api.provider your-llm-provider
+## 配置 LLM API 服务的 Key
+$ adb shell setprop jsar.threepio.api.key your-api-key
+## 配置 LLM API 服务地址 例如 `https://dashscope.aliyuncs.com/compatible-mode/v1`
+$ adb shell setprop jsar.threepio.api.endpoint your-api-endpoint
+```
+
+我们以[千问模型](https://github.com/QwenLM/Qwen)举例  
+```sh
+$ adb shell setprop jsar.threepio.api.modelid qwen-plus-latest
+$ adb shell setprop jsar.threepio.api.provider qwen
+$ adb shell setprop jsar.threepio.api.key your-api-key
+$ adb shell setprop jsar.threepio.api.endpoint https://dashscope.aliyuncs.com/compatible-mode/v1
+```
+
 ### 使用 Chrome DevTools 调试
 
 在应用进程日志中找到：

lib/bindings/env.ts

@@ -97,3 +97,35 @@ export function printSummary() {
  * Create an interval timer with 500ms to send keep alive update. It will be hangup when Script is busy.
  */
 setInterval(() => nativeContext.keepAlive(), 500);
+
+/**
+ * 
+ * @returns LLM API key
+ */
+export const getThreepioApiKey = (): string => {
+  return process.env.JSAR_THREEPIO_API_KEY as string;
+};
+
+/**
+ * 
+ * @returns LLM API provider name, such as 'doubao', 'qwen', etc.
+ */
+export const getThreepioApiProvider = (): string => {
+  return process.env.JSAR_THREEPIO_API_PROVIDER as string;
+};
+
+/**
+ * 
+ * @returns LLM API model ID, such as 'gpt-4', 'qwen-7b', etc.
+ */
+export const getThreepioApiModelId = (): string => {
+  return process.env.JSAR_THREEPIO_API_MODELID as string;
+};
+
+/**
+ * 
+ * @returns LLM API endpoint, such as 'https://api.doubao.com/v1/chat/completions', etc.
+ */
+export const getThreepioApiEndpoint = (): string => {
+  return process.env.JSAR_THREEPIO_API_ENDPOINT as string;
+};

lib/runtime2/index.ts

@@ -10,6 +10,7 @@ import { ResourceLoaderOnTransmute } from './jsardom/ResourceLoader';
 import createModel3dViewer from './viewers/model3d';  // glb, gltf ...
 import createImage2dViewer from './viewers/image2d';  // png, jpg, etc ...
 import createSplineDesignViewer from './viewers/splinedesign';  // splinedesign
+import { Threepio } from './threepio';
 
 Object.defineProperty(BABYLON.PrecisionDate, 'Now', {
   get: () => getPerformanceNow(),
@@ -64,7 +65,7 @@ async function evaluateXSML(gl: WebGLRenderingContext | WebGL2RenderingContext,
 
 export class TransmuteRuntime2 extends EventTarget {
   #browsingContext: Transmute.BrowsingContext;
-
+  #threepio: Threepio;
   constructor(private gl: WebGLRenderingContext | WebGL2RenderingContext, private id: number) {
     super();
     {
@@ -86,13 +87,18 @@ export class TransmuteRuntime2 extends EventTarget {
       const browsingContext = new BrowsingContext();
       browsingContext.setResourceLoader(new ResourceLoaderOnTransmute());
       this.#browsingContext = browsingContext;
+      this.#threepio = new Threepio(browsingContext);
     }
     this.dispatchEvent(new Event('rendererReady'));
   }
 
-  async start(url: string) {
-    console.info(`Content(#${this.id}): receiving a document request: ${url}`);
-    await this.load(url);
+  async start(input: string) {
+    console.info(`Content(#${this.id}): receiving a document request: ${input}`);
+    if (input.startsWith('http:') || input.startsWith('https:') || input.startsWith('/')) {
+      await this.load(input);
+    } else {
+      await this.#threepio.request(input);
+    }
   }
 
   private async load(codeOrUrl: string, urlBase?: string) {

lib/runtime2/threepio/api/index.ts

@@ -0,0 +1,33 @@
+import type { Anthropic } from '@anthropic-ai/sdk';
+import { ApiConfiguration, MoudleInfo } from '../shared/api';
+import { ApiStream, ApiStreamUsageChunk } from './transform/stream';
+import { DoubaoHandler } from './providers/doubao';
+import { QwenHandler } from './providers/qwen';
+
+export type LLMMessageParam = Anthropic.Messages.MessageParam;
+
+/**
+ * Interface for defining APIs to a specific large language model on cloud.
+ * @interface ApiHandler
+ * @returns A promise that resolves to an `ApiStreamUsageChunk` object containing usage data, or `undefined` if not available.
+ */
+export interface ApiHandler {
+  //`createMessage` is used to send a system prompt and a list of messages to the model, returning a streaming response.
+  createMessage(systemPrompt: string, messages: LLMMessageParam[]): ApiStream;
+  // retrieves the model's identifier and related information.
+  getModel(): { id: string; info: MoudleInfo };
+  // (optional) provides usage statistics for the API stream, if available.
+  getApiStreamUsage?(): Promise<ApiStreamUsageChunk | undefined>;
+}
+
+export function buildApiHandler(configuration: ApiConfiguration): ApiHandler {
+  const { apiProvider, ...options } = configuration;
+  switch (apiProvider) {
+    case 'doubao':
+      return new DoubaoHandler(options);
+    case 'qwen':
+      return new QwenHandler(options);
+    default:
+      throw new Error(`Unsupported API provider: ${apiProvider}`);
+  }
+}

lib/runtime2/threepio/api/providers/doubao.ts

@@ -0,0 +1,71 @@
+import OpenAI from 'openai';
+import { ApiHandler, LLMMessageParam } from '..';
+import {
+  MoudleInfo,
+  LLMModelId,
+  ApiHandlerOptions,
+  LLMModelToParameters,
+} from '../../shared/api';
+import { convertToOpenAiMessages } from '../transform/openaiFormat';
+import { ApiStream } from '../transform/stream';
+
+export class DoubaoHandler implements ApiHandler {
+  #options: ApiHandlerOptions;
+  #client: OpenAI;
+
+  constructor(options: ApiHandlerOptions) {
+    this.#options = options;
+    this.#client = new OpenAI({
+      baseURL: options.endpoint,
+      apiKey: options.apiKey,
+    });
+  }
+
+  getModel(): { id: LLMModelId; info: MoudleInfo } {
+    const modelId = this.#options.apiModelId;
+    if (modelId in LLMModelToParameters) {
+      const id = modelId as LLMModelId;
+      return { id, info: LLMModelToParameters[id] };
+    } else {
+      throw new Error(`Model ID ${modelId} is not supported.`);
+    }
+  }
+
+  async *createMessage(systemPrompt: string, messages: LLMMessageParam[]): ApiStream {
+    const model = this.getModel();
+    const openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
+      { role: 'system', content: systemPrompt },
+      ...convertToOpenAiMessages(messages),
+    ];
+    const stream = await this.#client.chat.completions.create({
+      model: model.id,
+      max_completion_tokens: model.info.max_completion_tokens,
+      messages: openAiMessages,
+      stream: true,
+      stream_options: { include_usage: true },
+      temperature: 0,
+    });
+
+    for await (const chunk of stream) {
+      const delta = chunk.choices[0]?.delta;
+      if (delta?.content) {
+        yield {
+          type: 'text',
+          text: delta.content,
+        };
+      }
+
+      if (chunk.usage) {
+        yield {
+          type: 'usage',
+          inputTokens: chunk.usage.prompt_tokens || 0,
+          outputTokens: chunk.usage.completion_tokens || 0,
+          // @ts-ignore-next-line
+          cacheReadTokens: chunk.usage.prompt_cache_hit_tokens || 0,
+          // @ts-ignore-next-line
+          cacheWriteTokens: chunk.usage.prompt_cache_miss_tokens || 0,
+        };
+      }
+    }
+  }
+}

lib/runtime2/threepio/api/providers/qwen.ts

@@ -0,0 +1,72 @@
+import OpenAI from 'openai';
+import { ApiHandler, LLMMessageParam } from '..';
+import {
+  ApiHandlerOptions,
+  LLMModelToParameters,
+  MoudleInfo,
+  LLMModelId
+} from '../../shared/api';
+import { convertToOpenAiMessages } from '../transform/openaiFormat';
+import { ApiStream } from '../transform/stream';
+
+export class QwenHandler implements ApiHandler {
+  #options: ApiHandlerOptions;
+  #client: OpenAI;
+
+  constructor(options: ApiHandlerOptions) {
+    this.#options = options;
+    this.#client = new OpenAI({
+      baseURL: options.endpoint,
+      apiKey: options.apiKey,
+    });
+  }
+
+  getModel(): { id: LLMModelId; info: MoudleInfo } {
+    const modelId = this.#options.apiModelId as LLMModelId;
+    if (!modelId) {
+      throw new Error('Model ID is not provided in the options.');
+    }
+    return {
+      id: modelId,
+      info: LLMModelToParameters[modelId],
+    }
+  }
+
+  async *createMessage(systemPrompt: string, messages: LLMMessageParam[]): ApiStream {
+    const model = this.getModel();
+    const openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
+      { role: 'system', content: systemPrompt },
+      ...convertToOpenAiMessages(messages),
+    ];
+    const stream = await this.#client.chat.completions.create({
+      model: model.id,
+      max_completion_tokens: model.info.max_completion_tokens,
+      messages: openAiMessages,
+      stream: true,
+      stream_options: { include_usage: true },
+      ...({ temperature: 0 }),
+    });
+
+    for await (const chunk of stream) {
+      const delta = chunk.choices[0]?.delta;
+      if (delta?.content) {
+        yield {
+          type: 'text',
+          text: delta.content,
+        };
+      }
+
+      if (chunk.usage) {
+        yield {
+          type: 'usage',
+          inputTokens: chunk.usage.prompt_tokens || 0,
+          outputTokens: chunk.usage.completion_tokens || 0,
+          // @ts-ignore-next-line
+          cacheReadTokens: chunk.usage.prompt_cache_hit_tokens || 0,
+          // @ts-ignore-next-line
+          cacheWriteTokens: chunk.usage.prompt_cache_miss_tokens || 0,
+        };
+      }
+    }
+  }
+}