Skip to content

Commit 3066f0a

Browse files
haraldschillyharaldschilly
committed
server/llm: finishing v0.2 API update, getting rid of "langchain" pkg and centralizing the history reconstruction
1 parent ff2ed35 commit 3066f0a

File tree

8 files changed

+163
-410
lines changed

8 files changed

+163
-410
lines changed

‎src/packages/pnpm-lock.yaml

Lines changed: 31 additions & 252 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/packages/server/llm/anthropic.ts

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import {
44
MessagesPlaceholder,
55
} from "@langchain/core/prompts";
66
import { RunnableWithMessageHistory } from "@langchain/core/runnables";
7-
import { ChatMessageHistory } from "langchain/stores/message/in_memory";
87

98
import getLogger from "@cocalc/backend/logger";
109
import { getServerSettings } from "@cocalc/database/settings";
@@ -14,7 +13,7 @@ import {
1413
isAnthropicModel,
1514
} from "@cocalc/util/db-schema/llm-utils";
1615
import { ChatOutput, History } from "@cocalc/util/types/llm";
17-
import { AIMessage, HumanMessage } from "@langchain/core/messages";
16+
import { transformHistoryToMessages } from "./chat-history";
1817
import { numTokens } from "./chatgpt-numtokens";
1918

2019
const log = getLogger("llm:anthropic");
@@ -93,22 +92,12 @@ export async function evaluateAnthropic(
9392
config: { configurable: { sessionId: "ignored" } },
9493
inputMessagesKey: "input",
9594
historyMessagesKey: "history",
96-
getMessageHistory: async (_) => {
97-
const chatHistory = new ChatMessageHistory();
98-
if (history) {
99-
let nextRole: "model" | "user" = "user";
100-
for (const { content } of history) {
101-
historyTokens += numTokens(content);
102-
if (nextRole === "user") {
103-
await chatHistory.addMessage(new HumanMessage(content));
104-
} else {
105-
await chatHistory.addMessage(new AIMessage(content));
106-
}
107-
nextRole = nextRole === "user" ? "model" : "user";
108-
}
109-
}
110-
111-
return chatHistory;
95+
getMessageHistory: async () => {
96+
const { messageHistory, tokens } = await transformHistoryToMessages(
97+
history,
98+
);
99+
historyTokens = tokens;
100+
return messageHistory;
112101
},
113102
});
114103

‎src/packages/server/llm/chat-history.ts

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { InMemoryChatMessageHistory } from "@langchain/core/chat_history";
2+
import { AIMessage, HumanMessage } from "@langchain/core/messages";
3+
4+
import { History } from "@cocalc/util/types/llm";
5+
import { numTokens } from "./chatgpt-numtokens";
6+
7+
// reconstruct the chat history from CoCalc's data
8+
// TODO: must be robust for repeated messages from the same user and ending in an assistant message
9+
export async function transformHistoryToMessages(
10+
history?: History,
11+
): Promise<{ messageHistory: InMemoryChatMessageHistory; tokens: number }> {
12+
let tokens = 0;
13+
14+
const messageHistory = new InMemoryChatMessageHistory();
15+
if (history) {
16+
let nextRole: "model" | "user" = "user";
17+
for (const { content } of history) {
18+
tokens += numTokens(content);
19+
if (nextRole === "user") {
20+
await messageHistory.addMessage(new HumanMessage(content));
21+
} else {
22+
await messageHistory.addMessage(new AIMessage(content));
23+
}
24+
nextRole = nextRole === "user" ? "model" : "user";
25+
}
26+
}
27+
28+
return { messageHistory, tokens };
29+
}

‎src/packages/server/llm/custom-openai.ts

Lines changed: 78 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,101 +1,93 @@
11
import {
2-
ChatPromptTemplate,
3-
MessagesPlaceholder,
4-
} from "@langchain/core/prompts";
5-
import { RunnableWithMessageHistory } from "@langchain/core/runnables";
6-
import { ChatMessageHistory } from "langchain/stores/message/in_memory";
2+
ChatPromptTemplate,
3+
MessagesPlaceholder,
4+
} from "@langchain/core/prompts";
5+
import { RunnableWithMessageHistory } from "@langchain/core/runnables";
76

8-
import getLogger from "@cocalc/backend/logger";
9-
import { fromCustomOpenAIModel, isCustomOpenAI } from "@cocalc/util/db-schema/llm-utils";
10-
import { ChatOutput, History } from "@cocalc/util/types/llm";
11-
import { AIMessage, HumanMessage } from "@langchain/core/messages";
12-
import { numTokens } from "./chatgpt-numtokens";
13-
import { getCustomOpenAI } from "./client";
14-
15-
const log = getLogger("llm:custom_openai");
7+
import getLogger from "@cocalc/backend/logger";
8+
import {
9+
fromCustomOpenAIModel,
10+
isCustomOpenAI,
11+
} from "@cocalc/util/db-schema/llm-utils";
12+
import { ChatOutput, History } from "@cocalc/util/types/llm";
13+
import { transformHistoryToMessages } from "./chat-history";
14+
import { numTokens } from "./chatgpt-numtokens";
15+
import { getCustomOpenAI } from "./client";
1616

17-
// subset of ChatOptions, but model is a string
18-
interface CustomOpenAIOpts {
19-
input: string; // new input that user types
20-
system?: string; // extra setup that we add for relevance and context
21-
history?: History;
22-
model: string; // this must be custom_openai-[model]
23-
stream?: (output?: string) => void;
24-
maxTokens?: number;
25-
}
17+
const log = getLogger("llm:custom_openai");
2618

27-
export async function evaluateCustomOpenAI(
28-
opts: Readonly<CustomOpenAIOpts>,
29-
): Promise<ChatOutput> {
30-
if (!isCustomOpenAI(opts.model)) {
31-
throw new Error(`model ${opts.model} not supported`);
32-
}
33-
const model = fromCustomOpenAIModel(opts.model);
34-
const { system, history, input, maxTokens, stream } = opts;
35-
log.debug("evaluateCustomOpenAI", {
36-
input,
37-
history,
38-
system,
39-
model,
40-
stream: stream != null,
41-
maxTokens,
42-
});
19+
// subset of ChatOptions, but model is a string
20+
interface CustomOpenAIOpts {
21+
input: string; // new input that user types
22+
system?: string; // extra setup that we add for relevance and context
23+
history?: History;
24+
model: string; // this must be custom_openai-[model]
25+
stream?: (output?: string) => void;
26+
maxTokens?: number;
27+
}
4328

44-
const customOpenAI = await getCustomOpenAI(model);
29+
export async function evaluateCustomOpenAI(
30+
opts: Readonly<CustomOpenAIOpts>,
31+
): Promise<ChatOutput> {
32+
if (!isCustomOpenAI(opts.model)) {
33+
throw new Error(`model ${opts.model} not supported`);
34+
}
35+
const model = fromCustomOpenAIModel(opts.model);
36+
const { system, history, input, maxTokens, stream } = opts;
37+
log.debug("evaluateCustomOpenAI", {
38+
input,
39+
history,
40+
system,
41+
model,
42+
stream: stream != null,
43+
maxTokens,
44+
});
4545

46-
const prompt = ChatPromptTemplate.fromMessages([
47-
["system", system ?? ""],
48-
new MessagesPlaceholder("chat_history"),
49-
["human", "{input}"],
50-
]);
46+
const customOpenAI = await getCustomOpenAI(model);
5147

52-
const chain = prompt.pipe(customOpenAI);
48+
const prompt = ChatPromptTemplate.fromMessages([
49+
["system", system ?? ""],
50+
new MessagesPlaceholder("chat_history"),
51+
["human", "{input}"],
52+
]);
5353

54-
let historyTokens = 0;
54+
const chain = prompt.pipe(customOpenAI);
5555

56-
const chainWithHistory = new RunnableWithMessageHistory({
57-
runnable: chain,
58-
config: { configurable: { sessionId: "ignored" } },
59-
inputMessagesKey: "input",
60-
historyMessagesKey: "chat_history",
61-
getMessageHistory: async (_) => {
62-
const chatHistory = new ChatMessageHistory();
63-
if (history) {
64-
let nextRole: "model" | "user" = "user";
65-
for (const { content } of history) {
66-
historyTokens = numTokens(content);
67-
if (nextRole === "user") {
68-
await chatHistory.addMessage(new HumanMessage(content));
69-
} else {
70-
await chatHistory.addMessage(new AIMessage(content));
71-
}
72-
nextRole = nextRole === "user" ? "model" : "user";
73-
}
74-
}
56+
let historyTokens = 0;
7557

76-
return chatHistory;
77-
},
78-
});
58+
const chainWithHistory = new RunnableWithMessageHistory({
59+
runnable: chain,
60+
config: { configurable: { sessionId: "ignored" } },
61+
inputMessagesKey: "input",
62+
historyMessagesKey: "chat_history",
63+
getMessageHistory: async () => {
64+
const { messageHistory, tokens } = await transformHistoryToMessages(
65+
history,
66+
);
67+
historyTokens = tokens;
68+
return messageHistory;
69+
},
70+
});
7971

80-
const chunks = await chainWithHistory.stream({ input });
72+
const chunks = await chainWithHistory.stream({ input });
8173

82-
let output = "";
83-
for await (const chunk of chunks) {
84-
output += chunk;
85-
opts.stream?.(chunk);
86-
}
74+
let output = "";
75+
for await (const chunk of chunks) {
76+
output += chunk;
77+
opts.stream?.(chunk);
78+
}
8779

88-
// and an empty call when done
89-
opts.stream?.();
80+
// and an empty call when done
81+
opts.stream?.();
9082

91-
// we use that GPT3 tokenizer to get an approximate number of tokens
92-
const prompt_tokens = numTokens(input) + historyTokens;
93-
const completion_tokens = numTokens(output);
83+
// we use that GPT3 tokenizer to get an approximate number of tokens
84+
const prompt_tokens = numTokens(input) + historyTokens;
85+
const completion_tokens = numTokens(output);
9486

95-
return {
96-
output,
97-
total_tokens: prompt_tokens + completion_tokens,
98-
completion_tokens,
99-
prompt_tokens,
100-
};
101-
}
87+
return {
88+
output,
89+
total_tokens: prompt_tokens + completion_tokens,
90+
completion_tokens,
91+
prompt_tokens,
92+
};
93+
}

‎src/packages/server/llm/google-genai-client.ts

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,12 @@
55
*/
66

77
import { GenerativeModel, GoogleGenerativeAI } from "@google/generative-ai";
8-
import { AIMessage, HumanMessage } from "@langchain/core/messages";
98
import {
109
ChatPromptTemplate,
1110
MessagesPlaceholder,
1211
} from "@langchain/core/prompts";
1312
import { RunnableWithMessageHistory } from "@langchain/core/runnables";
1413
import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
15-
import { ChatMessageHistory } from "langchain/stores/message/in_memory";
1614

1715
import getLogger from "@cocalc/backend/logger";
1816
import { getServerSettings } from "@cocalc/database/settings";
@@ -23,6 +21,7 @@ import {
2321
isGoogleModel,
2422
} from "@cocalc/util/db-schema/llm-utils";
2523
import { ChatOutput, History } from "@cocalc/util/types/llm";
24+
import { transformHistoryToMessages } from "./chat-history";
2625

2726
const log = getLogger("llm:google-genai");
2827

@@ -133,21 +132,9 @@ export class GoogleGenAIClient {
133132
config: { configurable: { sessionId: "ignored" } },
134133
inputMessagesKey: "input",
135134
historyMessagesKey: "history",
136-
getMessageHistory: async (_) => {
137-
const chatHistory = new ChatMessageHistory();
138-
if (history) {
139-
let nextRole: "model" | "user" = "user";
140-
for (const { content } of history) {
141-
if (nextRole === "user") {
142-
await chatHistory.addMessage(new HumanMessage(content));
143-
} else {
144-
await chatHistory.addMessage(new AIMessage(content));
145-
}
146-
nextRole = nextRole === "user" ? "model" : "user";
147-
}
148-
}
149-
150-
return chatHistory;
135+
getMessageHistory: async () => {
136+
const { messageHistory } = await transformHistoryToMessages(history);
137+
return messageHistory;
151138
},
152139
});
153140

‎src/packages/server/llm/mistral.ts

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,12 @@ import {
44
} from "@langchain/core/prompts";
55
import { RunnableWithMessageHistory } from "@langchain/core/runnables";
66
import { ChatMistralAI } from "@langchain/mistralai";
7-
import { ChatMessageHistory } from "langchain/stores/message/in_memory";
8-
import { AIMessage, HumanMessage } from "@langchain/core/messages";
97

108
import getLogger from "@cocalc/backend/logger";
119
import { getServerSettings } from "@cocalc/database/settings";
1210
import { isMistralModel } from "@cocalc/util/db-schema/llm-utils";
1311
import { ChatOutput, History } from "@cocalc/util/types/llm";
12+
import { transformHistoryToMessages } from "./chat-history";
1413
import { numTokens } from "./chatgpt-numtokens";
1514

1615
const log = getLogger("llm:mistral");
@@ -72,22 +71,12 @@ export async function evaluateMistral(
7271
config: { configurable: { sessionId: "ignored" } },
7372
inputMessagesKey: "input",
7473
historyMessagesKey: "history",
75-
getMessageHistory: async (_) => {
76-
const chatHistory = new ChatMessageHistory();
77-
if (history) {
78-
let nextRole: "model" | "user" = "user";
79-
for (const { content } of history) {
80-
historyTokens += numTokens(content);
81-
if (nextRole === "user") {
82-
await chatHistory.addMessage(new HumanMessage(content));
83-
} else {
84-
await chatHistory.addMessage(new AIMessage(content));
85-
}
86-
nextRole = nextRole === "user" ? "model" : "user";
87-
}
88-
}
89-
90-
return chatHistory;
74+
getMessageHistory: async () => {
75+
const { messageHistory, tokens } = await transformHistoryToMessages(
76+
history,
77+
);
78+
historyTokens = tokens;
79+
return messageHistory;
9180
},
9281
});
9382

‎src/packages/server/llm/ollama.ts

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,11 @@ import {
33
MessagesPlaceholder,
44
} from "@langchain/core/prompts";
55
import { RunnableWithMessageHistory } from "@langchain/core/runnables";
6-
import { ChatMessageHistory } from "langchain/stores/message/in_memory";
76

87
import getLogger from "@cocalc/backend/logger";
98
import { fromOllamaModel, isOllamaLLM } from "@cocalc/util/db-schema/llm-utils";
109
import { ChatOutput, History } from "@cocalc/util/types/llm";
11-
import { AIMessage, HumanMessage } from "@langchain/core/messages";
10+
import { transformHistoryToMessages } from "./chat-history";
1211
import { numTokens } from "./chatgpt-numtokens";
1312
import { getOllama } from "./client";
1413

@@ -58,22 +57,12 @@ export async function evaluateOllama(
5857
config: { configurable: { sessionId: "ignored" } },
5958
inputMessagesKey: "input",
6059
historyMessagesKey: "chat_history",
61-
getMessageHistory: async (_) => {
62-
const chatHistory = new ChatMessageHistory();
63-
if (history) {
64-
let nextRole: "model" | "user" = "user";
65-
for (const { content } of history) {
66-
historyTokens = numTokens(content);
67-
if (nextRole === "user") {
68-
await chatHistory.addMessage(new HumanMessage(content));
69-
} else {
70-
await chatHistory.addMessage(new AIMessage(content));
71-
}
72-
nextRole = nextRole === "user" ? "model" : "user";
73-
}
74-
}
75-
76-
return chatHistory;
60+
getMessageHistory: async () => {
61+
const { messageHistory, tokens } = await transformHistoryToMessages(
62+
history,
63+
);
64+
historyTokens = tokens;
65+
return messageHistory;
7766
},
7867
});
7968

‎src/packages/server/package.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,6 @@
9191
"json-stable-stringify": "^1.0.1",
9292
"jwt-decode": "^3.1.2",
9393
"lambda-cloud-node-api": "^1.0.1",
94-
"langchain": "^0.2.4",
9594
"libsodium-wrappers": "^0.7.13",
9695
"lodash": "^4.17.21",
9796
"lru-cache": "^7.14.1",

0 commit comments

Comments
 (0)