Skip to content

Commit 1246cdb

Browse files
csharpfritzcsharpfritz
authored
Merge branch 'dev' into dependabot/add-v2-config-file
2 parents 9ccf1ef + b7affc3 commit 1246cdb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+540
-975
lines changed

‎ConsoleChatbot/ConsoleChatbot.csproj

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net5.0</TargetFramework>
5+
<TargetFramework>net9.0</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>
@@ -16,11 +16,12 @@
1616
</ItemGroup>
1717

1818
<ItemGroup>
19-
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.2.0" />
20-
<PackageReference Include="Microsoft.Extensions.Logging" Version="2.1.1" />
21-
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.1.1" />
22-
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.2.0" />
23-
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="2.2.0" />
19+
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="9.0.0" />
20+
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />
21+
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0" />
22+
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.0" />
23+
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.0" />
24+
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
2425
</ItemGroup>
2526

2627
</Project>

‎ConsoleChatbot/Program.cs

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,17 +48,23 @@ private static FritzBot CreateFritzBot(IChatService chatService)
4848

4949
serviceCollection.AddHttpClient("ShoutoutCommand", c =>
5050
{
51-
c.BaseAddress = new Uri("https://api.twitch.tv/kraken/channels/");
5251
c.DefaultRequestHeaders.Add("client-id", config["StreamServices:Twitch:ClientId"]);
5352
});
5453

5554
FritzBot.RegisterCommands(serviceCollection);
55+
56+
var loggerService = LoggerFactory.Create(configure =>
57+
configure.AddSimpleConsole(options =>
58+
{
59+
options.IncludeScopes = true;
60+
})
61+
.SetMinimumLevel(LogLevel.Information)
62+
);
5663
var svcProvider = serviceCollection.BuildServiceProvider();
57-
var loggerFactory = svcProvider.GetService<ILoggerFactory>()
58-
.AddConsole(LogLevel.Information);
64+
var loggerFactory = svcProvider.GetService<ILoggerFactory>();
5965

6066
return new FritzBot(config, svcProvider, loggerFactory);
61-
67+
6268
}
6369

6470
}

‎Fritz.Chatbot/Commands/AddHatCommand.cs

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎Fritz.Chatbot/Commands/PredictHatCommand.cs

Lines changed: 104 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,115 +1,150 @@
1-
using Fritz.StreamLib.Core;
2-
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Prediction;
3-
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Prediction.Models;
1+
using Azure.AI.OpenAI;
2+
using Fritz.StreamLib.Core;
3+
using Fritz.StreamTools.Hubs;
4+
using Microsoft.AspNetCore.SignalR;
5+
using Microsoft.Extensions.AI;
46
using Microsoft.Extensions.Configuration;
57
using System;
6-
using System.Linq;
8+
using System.ClientModel;
79
using System.Collections.Generic;
8-
using System.Collections.Immutable;
9-
using System.Text;
10+
using System.Diagnostics;
11+
using System.IO;
1012
using System.Threading.Tasks;
11-
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training;
12-
using System.Net;
13-
using Microsoft.AspNetCore.SignalR;
14-
using Fritz.StreamTools.Hubs;
1513

1614
namespace Fritz.Chatbot.Commands
1715
{
1816
public class PredictHatCommand : IBasicCommand
1917
{
2018
public string Trigger => "hat";
21-
public string Description => "Identify which hat Fritz is wearing";
19+
public string Description => "Describe which hat Fritz is wearing";
2220
public TimeSpan? Cooldown => TimeSpan.FromSeconds(30);
2321

24-
private string _CustomVisionKey = "";
25-
private string _AzureEndpoint = "";
26-
private string _TwitchChannel = "";
27-
private Guid _AzureProjectId;
22+
private DateTimeOffset? _LastRun = DateTimeOffset.MinValue;
23+
private TimeSpan _LastRunCooldown = TimeSpan.FromMinutes(5);
24+
private HatMetadata _LastHatDetected = null!;
2825

29-
internal static string IterationName = "";
30-
private ScreenshotTrainingService _TrainHat;
31-
private readonly HatDescriptionRepository _Repository;
26+
private ScreenshotTrainingService _ObsConnection;
27+
//private readonly HatDescriptionRepository _Repository;
3228
private readonly IHubContext<ObsHub> _HubContext;
29+
private readonly string _AzureOpenAiEndpoint;
30+
private readonly string _AzureOpenAiKey;
31+
private readonly string _AzureOpenAiModel;
3332

34-
public PredictHatCommand(IConfiguration configuration, ScreenshotTrainingService service, HatDescriptionRepository repository, IHubContext<ObsHub> hubContext)
33+
public PredictHatCommand(IConfiguration configuration, ScreenshotTrainingService service, IHubContext<ObsHub> hubContext)
3534
{
36-
_CustomVisionKey = configuration["AzureServices:HatDetection:Key"];
37-
_AzureEndpoint = configuration["AzureServices:HatDetection:CustomVisionEndpoint"];
38-
_TwitchChannel = configuration["StreamServices:Twitch:Channel"];
39-
_AzureProjectId = Guid.Parse(configuration["AzureServices:HatDetection:ProjectId"]);
40-
_TrainHat = service;
41-
_Repository = repository;
35+
_AzureOpenAiEndpoint = configuration["AzureOpenAiEndpoint"];
36+
_AzureOpenAiKey = configuration["AzureOpenAiKey"];
37+
_AzureOpenAiModel = configuration["AzureOpenAiModel"];
38+
_ObsConnection = service;
4239
_HubContext = hubContext;
4340
}
4441

4542
public async Task Execute(IChatService chatService, string userName, ReadOnlyMemory<char> rhs)
4643
{
4744

48-
if (string.IsNullOrEmpty(IterationName)) {
49-
await IdentifyIterationName();
50-
}
51-
52-
var client = new CustomVisionPredictionClient()
45+
if (!userName.Equals("csharpfritz", StringComparison.InvariantCultureIgnoreCase)
46+
&& _LastRun.HasValue && DateTimeOffset.Now - _LastRun < _LastRunCooldown)
5347
{
54-
ApiKey = _CustomVisionKey,
55-
Endpoint = _AzureEndpoint,
56-
};
48+
// send a message about the last hat detected
49+
await chatService.SendMessageAsync($"@{userName} from my last analysis {Math.Round(DateTimeOffset.Now.Subtract(_LastRun.Value).TotalMinutes)} minutes ago I can tell you Fritz's hat is:");
50+
await chatService.SendMessageAsync(_LastHatDetected.Description);
51+
return;
52+
}
5753

58-
await _HubContext.Clients.All.SendAsync("shutter");
59-
var obsImage = await _TrainHat.GetScreenshotFromObs();
54+
var obsImage = await _ObsConnection.GetScreenshotFromObs();
6055

6156
////////////////////////////
6257

63-
ImagePrediction result;
58+
59+
HatMetadata hatDetected;
60+
6461
try
6562
{
66-
result = await client.DetectImageWithNoStoreAsync(_AzureProjectId, IterationName, obsImage);
67-
} catch (CustomVisionErrorException ex) {
63+
hatDetected = await PredictHat(obsImage);
64+
}
65+
catch
66+
{
67+
await chatService.SendMessageAsync("There was an error detecting this hat. Please try again in 30 seconds");
68+
return;
69+
}
6870

69-
71+
if (string.IsNullOrEmpty(hatDetected.Name))
72+
{
73+
await chatService.SendMessageAsync("There was an error detecting a hat. Please try again in 30 seconds");
74+
return;
75+
}
7076

71-
if (ex.Response.StatusCode == HttpStatusCode.NotFound) {
72-
await IdentifyIterationName();
73-
}
77+
// send a message about the hat detected
78+
await chatService.SendMessageAsync($"@{userName} I can tell you Fritz's hat is: {hatDetected.Name}");
79+
await chatService.SendMessageAsync(hatDetected.Description);
80+
await chatService.SendMessageAsync(hatDetected.Conclusion);
81+
_LastHatDetected = hatDetected;
82+
_LastRun = DateTimeOffset.Now;
7483

75-
await chatService.SendMessageAsync("Unable to detect Fritz's hat right now... please try again in 1 minute");
76-
return;
84+
}
7785

78-
}
86+
private async Task<HatMetadata> PredictHat(Stream obsImage)
87+
{
7988

80-
if (DateTime.UtcNow.Subtract(result.Created).TotalSeconds > Cooldown.Value.TotalSeconds) {
81-
await chatService.SendMessageAsync($"I previously predicted this hat about {DateTime.UtcNow.Subtract(result.Created).TotalSeconds} seconds ago");
82-
}
89+
var client = new AzureOpenAIClient(
90+
new Uri(_AzureOpenAiEndpoint),
91+
new ApiKeyCredential(_AzureOpenAiKey))
92+
.AsChatClient(_AzureOpenAiModel);
8393

84-
var bestMatch = result.Predictions.OrderByDescending(p => p.Probability).FirstOrDefault();
85-
if (bestMatch == null || bestMatch.Probability < 0.7D) {
86-
await chatService.SendMessageAsync("csharpAngry 404 Hat Not Found! Let's ask a moderator to !addhat so we can identify it next time");
87-
// do we store the image?
88-
return;
89-
}
94+
var systemPrompt =
95+
"""
96+
You are an AI assistant that can detect and describe baseball-style
97+
hats from an image. The user will provide an image of someone wearing a hat
98+
and they might be also wearing a headset. Ignore the headset and focus on the hat.
9099
91-
var hatData = (await _Repository.GetHatData(bestMatch.TagName));
92-
var nameToReport = (hatData == null ? bestMatch.TagName : (string.IsNullOrEmpty(hatData.Name) ? bestMatch.TagName : hatData.Name));
93-
await chatService.SendMessageAsync($"csharpClip I think (with {bestMatch.Probability.ToString("0.0%")} certainty) Jeff is currently wearing his {nameToReport} hat csharpClip");
94-
if (hatData != null && !string.IsNullOrEmpty(hatData.Description)) await chatService.SendMessageAsync(hatData.Description);
100+
This hat comes from a collection that includes hats that fall into one of these categories:
101+
Sports Teams, Colleges, Marvel Comics, Microsoft logos, Star Wars,
102+
Video Games, and other popular culture references.
95103
96-
await _HubContext.Clients.All.SendAsync("hatDetected", bestMatch.Probability.ToString("0.0%"), bestMatch.TagName, nameToReport, hatData?.Description);
104+
Be as descriptive as possible about the hat focusing on its design, colors,
105+
logo placement, text, category, organization that the hat references, and any notable features.
97106
98-
}
107+
Limit the description to 3 sentences.
108+
109+
Suggest a name for the hat based on the color, organization, or logo.
110+
111+
If the hat references a sports team or college, you should conclude with that school's slogan or cheer.
112+
113+
If the hat references a Marvel comic character, you should conclude with a comment about the hat in the tone of the references character or organization
114+
""";
99115

100-
private async Task IdentifyIterationName()
116+
var file = new byte[obsImage.Length];
117+
obsImage.ReadExactly(file, 0, (int)obsImage.Length);
118+
var messages = new List<ChatMessage>
101119
{
120+
new ChatMessage(ChatRole.System, systemPrompt),
121+
new ChatMessage(ChatRole.User, new AIContent[] {
122+
new ImageContent(file, "image/webp"), //
123+
new TextContent("Generate a description of the hat"), // Generate a description of the hat
124+
})
125+
};
102126

103-
var client = new CustomVisionTrainingClient() {
104-
ApiKey = _CustomVisionKey,
105-
Endpoint = _AzureEndpoint
106-
};
127+
var sw = Stopwatch.StartNew();
128+
var response = await client.CompleteAsync<HatMetadata>(messages, options: new ChatOptions { Temperature = 0.1f });
107129

108-
var iterations = await client.GetIterationsAsync(_AzureProjectId);
109-
IterationName = iterations
110-
.Where(i => !string.IsNullOrEmpty(i.PublishName) && i.Status == "Completed")
111-
.OrderByDescending(i => i.LastModified).First().PublishName;
130+
Console.WriteLine($"Elapsed: {sw.Elapsed}");
131+
Console.WriteLine($"Predicted hat: {response.Result}");
132+
return response.Result;
112133

113134
}
135+
114136
}
137+
138+
internal record HatMetadata(
139+
string Color,
140+
string Name,
141+
bool HasLogo,
142+
string? LogoShape,
143+
string Text,
144+
string Description,
145+
string Category,
146+
string LogoDescription,
147+
string Organization,
148+
string Conclusion);
149+
115150
}

‎Fritz.Chatbot/Commands/ResetHatAiCommand.cs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
using Fritz.StreamLib.Core;
22
using System;
3-
using System.Collections.Generic;
4-
using System.Text;
53
using System.Threading.Tasks;
64

75
namespace Fritz.Chatbot.Commands
@@ -18,7 +16,7 @@ public async Task Execute(IChatService chatService, string userName, bool isMode
1816

1917
if (!(isModerator || isBroadcaster)) return;
2018

21-
PredictHatCommand.IterationName = string.Empty;
19+
//PredictHatCommand.IterationName = string.Empty;
2220
await chatService.SendMessageAsync("Reset the AI iteration and will detect the latest for Hat identification next time !hat is called");
2321

2422
}

0 commit comments

Comments
 (0)