lite: support llm conat server endpoint

Author: williamstein

src/packages/lite/hub/llm.ts

@@ -0,0 +1,45 @@
+/*
+This is the lightweight analogue of @cocalc/server/llm
+*/
+
+import {
+  evaluateWithLangChain,
+  evaluateOllama,
+  heuristicNumTokens,
+} from "@cocalc/ai/llm";
+import { init as initConatLLM } from "@cocalc/conat/llm/server";
+import { isOllamaLLM } from "@cocalc/util/db-schema/llm-utils";
+import { listRows } from "./sqlite/database";
+
+function buildContext() {
+  const settings: Record<string, any> = {};
+  for (const row of listRows("server_settings") as {
+    name?: string;
+    value?: any;
+  }[]) {
+    if (row.name) {
+      settings[row.name] = row.value;
+    }
+  }
+
+  return {
+    settings: {
+      openai_api_key: settings.openai_api_key,
+      google_vertexai_key: settings.google_vertexai_key,
+      anthropic_api_key: settings.anthropic_api_key,
+      mistral_api_key: settings.mistral_api_key,
+    },
+    mode: "user" as const,
+    tokenCounter: heuristicNumTokens,
+  };
+}
+
+export async function init(): Promise<void> {
+  await initConatLLM(async (opts: any) => {
+    const context = buildContext();
+    if (isOllamaLLM(opts.model)) {
+      return evaluateOllama(opts);
+    }
+    return evaluateWithLangChain(opts, context);
+  });
+}

src/packages/lite/main.ts

@@ -28,6 +28,7 @@ import { localPathFileserver } from "@cocalc/backend/conat/files/local-path";
 import { init as initBugCounter } from "@cocalc/project/bug-counter";
 import { init as initChangefeeds } from "./hub/changefeeds";
 import { init as initHubApi } from "./hub/api";
+import { init as initLLM } from "./hub/llm";
 import { account_id } from "@cocalc/backend/data";
 import { init as initRemote } from "./remote";
 import { getAuthToken } from "./auth-token";
@@ -96,6 +97,9 @@ export async function main(): Promise<number> {
   logger.debug("start changefeed server");
   initChangefeeds({ client: conatClient });
 
+  logger.debug("start llm conat server");
+  await initLLM();
+
   const path = process.cwd();
 
   logger.debug("start hub api");

src/packages/lite/package.json

@@ -32,6 +32,7 @@
   "dependencies": {
     "@cocalc/assets": "workspace:*",
     "@cocalc/backend": "workspace:*",
+    "@cocalc/ai": "workspace:*",
     "@cocalc/conat": "workspace:*",
     "@cocalc/lite": "workspace:*",
     "@cocalc/project": "workspace:*",