Skip to content

Commit 7691a75

Browse files
committed
Fixed LLama.Web errors starting session
InferenceOptions.SamplingPipeline now is not required. Is not sent from client Added options to debug SignalR calls at appsettings.Development.json Changed default web model to llava-v1.6-mistral-7b.Q3_K_XS.gguf, previous one (llama-2-7b-chat.Q4_0.gguf) did no exist
1 parent 5e0c95f commit 7691a75

File tree

3 files changed

+7
-5
lines changed

3 files changed

+7
-5
lines changed

LLama.Web/Common/InferenceOptions.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,6 @@ public class InferenceOptions
2020
public IReadOnlyList<string> AntiPrompts { get; set; } = Array.Empty<string>();
2121

2222
/// <inheritdoc />
23-
public required ISamplingPipeline SamplingPipeline { get; set; }
23+
public ISamplingPipeline SamplingPipeline { get; set; } = new DefaultSamplingPipeline();
2424
}
2525
}

LLama.Web/appsettings.Development.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@
33
"Logging": {
44
"LogLevel": {
55
"Default": "Information",
6-
"Microsoft.AspNetCore": "Warning"
6+
"Microsoft.AspNetCore": "Warning",
7+
"Microsoft.AspNetCore.SignalR": "Debug",
8+
"Microsoft.AspNetCore.Http.Connections": "Debug"
79
}
810
}
911
}

LLama.Web/appsettings.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,13 @@
1010
"ModelLoadType": 0,
1111
"Models": [
1212
{
13-
"Name": "Example LLama2-7b-Chat",
13+
"Name": "Example LLava-v1.6-mistral",
1414
"MaxInstances": 20,
15-
"ModelPath": "..\\LLama.Unittest\\Models\\llama-2-7b-chat.Q4_0.gguf",
15+
"ModelPath": "..\\LLama.Unittest\\Models\\llava-v1.6-mistral-7b.Q3_K_XS.gguf",
1616
"ContextSize": 2048,
1717
"BatchSize": 2048,
1818
"Threads": 4,
19-
"GpuLayerCount": 6,
19+
"GpuLayerCount": 32,
2020
"UseMemorymap": true,
2121
"UseMemoryLock": false,
2222
"MainGpu": 0,

0 commit comments

Comments
 (0)