Skip to content

Commit 6e6f6ee

Browse files
committed
docs: update to use context api
1 parent ed4deba commit 6e6f6ee

File tree

1 file changed

+10
-8
lines changed

1 file changed

+10
-8
lines changed

README.md

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
2929

3030
const model = new LlamaModel({
3131
modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
32-
})
33-
const session = new LlamaChatSession({model});
32+
});
33+
const session = new LlamaChatSession({context: model.createContext()});
3434

3535

3636
const q1 = "Hi there, how are you?";
@@ -73,7 +73,7 @@ const model = new LlamaModel({
7373
modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin"),
7474
promptWrapper: new MyCustomChatPromptWrapper() // by default, LlamaChatPromptWrapper is used
7575
})
76-
const session = new LlamaChatSession({model});
76+
const session = new LlamaChatSession({context: model.createContext()});
7777

7878

7979
const q1 = "Hi there, how are you?";
@@ -98,29 +98,31 @@ import {LlamaModel, LlamaChatSession} from "node-llama-cpp";
9898

9999
const __dirname = path.dirname(fileURLToPath(import.meta.url));
100100

101-
const model = new LlamaChatSession({
101+
const model = new LlamaModel({
102102
modelPath: path.join(__dirname, "models", "vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin")
103103
});
104104

105+
const context = model.createContext();
106+
105107
const q1 = "Hi there, how are you?";
106108
console.log("AI: " + q1);
107109

108-
const tokens = model.encode(q1);
110+
const tokens = context.encode(q1);
109111
const res: number[] = [];
110-
for await (const chunk of model.evaluate(tokens)) {
112+
for await (const chunk of context.evaluate(tokens)) {
111113
res.push(chunk);
112114

113115
// it's important to not concatinate the results as strings,
114116
// as doing so will break some characters (like some emojis) that are made of multiple tokens.
115117
// by using an array of tokens, we can decode them correctly together.
116-
const resString: string = model.decode(Uint32Array.from(res));
118+
const resString: string = context.decode(Uint32Array.from(res));
117119

118120
const lastPart = resString.split("ASSISTANT:").reverse()[0];
119121
if (lastPart.includes("USER:"))
120122
break;
121123
}
122124

123-
const a1 = model.decode(Uint32Array.from(res)).split("USER:")[0];
125+
const a1 = context.decode(Uint32Array.from(res)).split("USER:")[0];
124126
console.log("AI: " + a1);
125127
```
126128

0 commit comments

Comments
 (0)