Skip to content

Commit b57789d

Browse files
csharpfritzcsharpfritz
committed
Added hat detection features
1 parent 78587eb commit b57789d

File tree

4 files changed

+142
-0
lines changed

4 files changed

+142
-0
lines changed

‎Fritz.Chatbot/Commands/AddHatCommand.cs

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
using Fritz.StreamLib.Core;
2+
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training;
3+
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training.Models;
4+
using Microsoft.Extensions.Configuration;
5+
using System;
6+
using System.Collections.Generic;
7+
using System.Text;
8+
using System.Threading.Tasks;
9+
10+
11+
namespace Fritz.Chatbot.Commands
12+
{
13+
public class AddHatCommand : IBasicCommand2
14+
{
15+
public string Trigger => "addhat";
16+
public string Description => "Moderators can add a screenshot of the stream to the image detection library";
17+
public TimeSpan? Cooldown => TimeSpan.FromMinutes(2);
18+
19+
private string _CustomVisionKey = "";
20+
private string _AzureEndpoint = "";
21+
private string _TwitchChannel = "";
22+
private Guid _AzureProjectId;
23+
24+
public AddHatCommand(IConfiguration configuration)
25+
{
26+
_CustomVisionKey = configuration["AzureServices:HatDetection:Key"];
27+
_AzureEndpoint = configuration["AzureServices:HatDetection:CustomVisionEndpoint"];
28+
_TwitchChannel = configuration["StreamServices:Twitch:Channel"];
29+
_AzureProjectId = Guid.Parse(configuration["AzureServices:HatDetection:ProjectId"]);
30+
}
31+
32+
public string TwitchScreenshotUrl => $"https://static-cdn.jtvnw.net/previews-ttv/live_user_{_TwitchChannel}-1280x720.jpg?_=";
33+
34+
public async Task Execute(IChatService chatService, string userName, bool isModerator, bool isVip, bool isBroadcaster, ReadOnlyMemory<char> rhs)
35+
{
36+
if (!(isModerator || isBroadcaster || isVip)) return;
37+
var trainingClient = new CustomVisionTrainingClient()
38+
{
39+
ApiKey = _CustomVisionKey,
40+
Endpoint = _AzureEndpoint
41+
};
42+
43+
await trainingClient.CreateImagesFromUrlsAsync(_AzureProjectId, new ImageUrlCreateBatch(
44+
new List<ImageUrlCreateEntry> {
45+
new ImageUrlCreateEntry(TwitchScreenshotUrl + Guid.NewGuid().ToString())
46+
}
47+
));
48+
49+
}
50+
51+
public Task Execute(IChatService chatService, string userName, ReadOnlyMemory<char> rhs)
52+
{
53+
throw new NotImplementedException();
54+
}
55+
}
56+
}

‎Fritz.Chatbot/Commands/PredictHatCommand.cs

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
using Fritz.StreamLib.Core;
2+
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Prediction;
3+
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Prediction.Models;
4+
using Microsoft.Extensions.Configuration;
5+
using System;
6+
using System.Linq;
7+
using System.Collections.Generic;
8+
using System.Collections.Immutable;
9+
using System.Text;
10+
using System.Threading.Tasks;
11+
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training;
12+
13+
namespace Fritz.Chatbot.Commands
14+
{
15+
public class PredictHatCommand : IBasicCommand
16+
{
17+
public string Trigger => "hat";
18+
public string Description => "Identify which hat Fritz is wearing";
19+
public TimeSpan? Cooldown => TimeSpan.FromSeconds(30);
20+
21+
private string _CustomVisionKey = "";
22+
private string _AzureEndpoint = "";
23+
private string _TwitchChannel = "";
24+
private Guid _AzureProjectId;
25+
26+
private static string _IterationName = "";
27+
28+
public PredictHatCommand(IConfiguration configuration)
29+
{
30+
_CustomVisionKey = configuration["AzureServices:HatDetection:Key"];
31+
_AzureEndpoint = configuration["AzureServices:HatDetection:CustomVisionEndpoint"];
32+
_TwitchChannel = configuration["StreamServices:Twitch:Channel"];
33+
_AzureProjectId = Guid.Parse(configuration["AzureServices:HatDetection:ProjectId"]);
34+
}
35+
36+
public string TwitchScreenshotUrl => $"https://static-cdn.jtvnw.net/previews-ttv/live_user_{_TwitchChannel}-1280x720.jpg?_=";
37+
38+
39+
public async Task Execute(IChatService chatService, string userName, ReadOnlyMemory<char> rhs)
40+
{
41+
42+
if (string.IsNullOrEmpty(_IterationName)) {
43+
await IdentifyIterationName();
44+
}
45+
46+
var client = new CustomVisionPredictionClient()
47+
{
48+
ApiKey = _CustomVisionKey,
49+
Endpoint = _AzureEndpoint
50+
};
51+
var result = await client.DetectImageUrlWithNoStoreAsync(_AzureProjectId, _IterationName, new ImageUrl(TwitchScreenshotUrl));
52+
53+
var bestMatch = result.Predictions.OrderByDescending(p => p.Probability).FirstOrDefault();
54+
if (bestMatch == null || bestMatch.Probability <= 0.3D) {
55+
await chatService.SendMessageAsync("No match found for the current hat");
56+
// do we store the image?
57+
return;
58+
}
59+
60+
await chatService.SendMessageAsync($"I think (with {bestMatch.Probability.ToString("0.0%")} certainty) Jeff is currently wearing his {bestMatch.TagName} hat");
61+
62+
}
63+
64+
private async Task IdentifyIterationName()
65+
{
66+
67+
var client = new CustomVisionTrainingClient() {
68+
ApiKey = _CustomVisionKey,
69+
Endpoint = _AzureEndpoint
70+
};
71+
72+
var iterations = await client.GetIterationsAsync(_AzureProjectId);
73+
_IterationName = iterations
74+
.Where(i => !string.IsNullOrEmpty(i.PublishName) && i.Status == "Completed")
75+
.OrderByDescending(i => i.LastModified).First().PublishName;
76+
77+
}
78+
}
79+
}

‎Fritz.Chatbot/Fritz.Chatbot.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
<PackageReference Include="Markdig" Version="0.18.3" />
1313
<PackageReference Include="Microsoft.AspNetCore.SignalR" Version="1.0.4" />
1414
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="1.0.4" />
15+
<PackageReference Include="Microsoft.Azure.CognitiveServices.Vision.CustomVision.Prediction" Version="1.0.0" />
16+
<PackageReference Include="Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training" Version="1.0.0" />
1517
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="2.2.0" />
1618
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="2.1.1" />
1719
<PackageReference Include="Microsoft.Extensions.Http" Version="2.1.1" />

‎Fritz.StreamTools/appsettings.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,11 @@
232232
"Key": ""
233233
},
234234
"AzureServices": {
235+
"HatDetection": {
236+
"Key": "<< SOMETHING I'M NOT TYPING HERE AGAIN>>",
237+
"CustomVisionEndpoint": "https://eastus2.api.cognitive.microsoft.com/",
238+
"ProjectId": "fbc689d9-79a6-4352-b2e4-320b327d6bd5"
239+
}
235240
},
236241
"GitHub": {
237242
"RepositoryName": "Fritz.StreamTools",

0 commit comments

Comments
 (0)