@@ -29,8 +29,8 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
29
29
30
30
const model = new LlamaModel ({
31
31
modelPath: path .join (__dirname , " models" , " vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin" )
32
- })
33
- const session = new LlamaChatSession ({model });
32
+ });
33
+ const session = new LlamaChatSession ({context: model . createContext () });
34
34
35
35
36
36
const q1 = " Hi there, how are you?" ;
@@ -73,7 +73,7 @@ const model = new LlamaModel({
73
73
modelPath: path .join (__dirname , " models" , " vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin" ),
74
74
promptWrapper: new MyCustomChatPromptWrapper () // by default, LlamaChatPromptWrapper is used
75
75
})
76
- const session = new LlamaChatSession ({model });
76
+ const session = new LlamaChatSession ({context: model . createContext () });
77
77
78
78
79
79
const q1 = " Hi there, how are you?" ;
@@ -98,29 +98,31 @@ import {LlamaModel, LlamaChatSession} from "node-llama-cpp";
98
98
99
99
const __dirname = path .dirname (fileURLToPath (import .meta .url ));
100
100
101
- const model = new LlamaChatSession ({
101
+ const model = new LlamaModel ({
102
102
modelPath: path .join (__dirname , " models" , " vicuna-13b-v1.5-16k.ggmlv3.q5_1.bin" )
103
103
});
104
104
105
+ const context = model .createContext ();
106
+
105
107
const q1 = " Hi there, how are you?" ;
106
108
console .log (" AI: " + q1 );
107
109
108
- const tokens = model .encode (q1 );
110
+ const tokens = context .encode (q1 );
109
111
const res: number [] = [];
110
- for await (const chunk of model .evaluate (tokens )) {
112
+ for await (const chunk of context .evaluate (tokens )) {
111
113
res .push (chunk );
112
114
113
115
// it's important to not concatinate the results as strings,
114
116
// as doing so will break some characters (like some emojis) that are made of multiple tokens.
115
117
// by using an array of tokens, we can decode them correctly together.
116
- const resString: string = model .decode (Uint32Array .from (res ));
118
+ const resString: string = context .decode (Uint32Array .from (res ));
117
119
118
120
const lastPart = resString .split (" ASSISTANT:" ).reverse ()[0 ];
119
121
if (lastPart .includes (" USER:" ))
120
122
break ;
121
123
}
122
124
123
- const a1 = model .decode (Uint32Array .from (res )).split (" USER:" )[0 ];
125
+ const a1 = context .decode (Uint32Array .from (res )).split (" USER:" )[0 ];
124
126
console .log (" AI: " + a1 );
125
127
```
126
128
0 commit comments