Skip to content

Commit 798f430

Browse files
csharpfritzcsharpfritz
committed
Added trainhat feature
1 parent c2a4f5f commit 798f430

File tree

8 files changed

+263
-26
lines changed

8 files changed

+263
-26
lines changed

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,16 @@ Fritz.StreamTools/wwwroot/lib/
99
.vscode/
1010
package-lock.json
1111

12+
#Ignore the lib folder
13+
lib/
14+
1215
# User-specific files
1316
*.suo
1417
*.user
1518
*.userosscache
1619
*.sln.docstates
20+
serviceDependencies.json
21+
serviceDependencies.local.json
1722

1823
# User-specific files (MonoDevelop/Xamarin Studio)
1924
*.userprefs

‎Fritz.Chatbot/Commands/AddHatCommand.cs‎

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -12,39 +12,22 @@ namespace Fritz.Chatbot.Commands
1212
{
1313
public class AddHatCommand : IBasicCommand2
1414
{
15+
private readonly ITrainHat _TrainHat;
16+
1517
public string Trigger => "addhat";
1618
public string Description => "Moderators can add a screenshot of the stream to the image detection library";
1719
public TimeSpan? Cooldown => TimeSpan.FromMinutes(2);
1820

19-
private string _CustomVisionKey = "";
20-
private string _AzureEndpoint = "";
21-
private string _TwitchChannel = "";
22-
private Guid _AzureProjectId;
23-
24-
public AddHatCommand(IConfiguration configuration)
21+
public AddHatCommand(ITrainHat trainHat)
2522
{
26-
_CustomVisionKey = configuration["AzureServices:HatDetection:Key"];
27-
_AzureEndpoint = configuration["AzureServices:HatDetection:CustomVisionEndpoint"];
28-
_TwitchChannel = configuration["StreamServices:Twitch:Channel"];
29-
_AzureProjectId = Guid.Parse(configuration["AzureServices:HatDetection:ProjectId"]);
23+
_TrainHat = trainHat;
3024
}
3125

32-
public string TwitchScreenshotUrl => $"https://static-cdn.jtvnw.net/previews-ttv/live_user_{_TwitchChannel}-1280x720.jpg?_=";
33-
3426
public async Task Execute(IChatService chatService, string userName, bool isModerator, bool isVip, bool isBroadcaster, ReadOnlyMemory<char> rhs)
3527
{
3628
if (!(isModerator || isBroadcaster || isVip)) return;
37-
var trainingClient = new CustomVisionTrainingClient()
38-
{
39-
ApiKey = _CustomVisionKey,
40-
Endpoint = _AzureEndpoint
41-
};
42-
43-
await trainingClient.CreateImagesFromUrlsAsync(_AzureProjectId, new ImageUrlCreateBatch(
44-
new List<ImageUrlCreateEntry> {
45-
new ImageUrlCreateEntry(TwitchScreenshotUrl + Guid.NewGuid().ToString())
46-
}
47-
));
29+
await _TrainHat.AddScreenshot();
30+
await chatService.SendMessageAsync("csharpCat Taking a screenshot and adding image to the knowledgebase");
4831

4932
}
5033

‎Fritz.Chatbot/Commands/HttpPageTitleCommand.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ private async Task Execute(string userName, string fullCommandText)
5151

5252
foreach (var url in urls)
5353
{
54+
55+
if (url.Equals("asp.net", StringComparison.InvariantCultureIgnoreCase)) continue;
56+
5457
var source = GetSource(url);
5558
if (string.IsNullOrEmpty(source))
5659
{

‎Fritz.Chatbot/Commands/PredictHatCommand.cs‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using System.Text;
1010
using System.Threading.Tasks;
1111
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training;
12+
using System.Net;
1213

1314
namespace Fritz.Chatbot.Commands
1415
{
@@ -48,16 +49,34 @@ public async Task Execute(IChatService chatService, string userName, ReadOnlyMem
4849
ApiKey = _CustomVisionKey,
4950
Endpoint = _AzureEndpoint
5051
};
51-
var result = await client.DetectImageUrlWithNoStoreAsync(_AzureProjectId, _IterationName, new ImageUrl(TwitchScreenshotUrl));
52+
53+
54+
ImagePrediction result;
55+
try
56+
{
57+
result = await client.DetectImageUrlWithNoStoreAsync(_AzureProjectId, _IterationName, new ImageUrl(TwitchScreenshotUrl));
58+
} catch (CustomVisionErrorException ex) {
59+
60+
61+
62+
if (ex.Response.StatusCode == HttpStatusCode.NotFound) {
63+
await IdentifyIterationName();
64+
}
65+
66+
await chatService.SendMessageAsync("Unable to detect Fritz's hat right now... please try again in 1 minute");
67+
return;
68+
69+
}
70+
5271

5372
var bestMatch = result.Predictions.OrderByDescending(p => p.Probability).FirstOrDefault();
5473
if (bestMatch == null || bestMatch.Probability <= 0.3D) {
55-
await chatService.SendMessageAsync("No match found for the current hat");
74+
await chatService.SendMessageAsync("csharpAngry 404 Hat Not Found! Let's ask a moderator to !addhat so we can identify it next time");
5675
// do we store the image?
5776
return;
5877
}
5978

60-
await chatService.SendMessageAsync($"I think (with {bestMatch.Probability.ToString("0.0%")} certainty) Jeff is currently wearing his {bestMatch.TagName} hat");
79+
await chatService.SendMessageAsync($"csharpClip I think (with {bestMatch.Probability.ToString("0.0%")} certainty) Jeff is currently wearing his {bestMatch.TagName} hat csharpClip");
6180

6281
}
6382

‎Fritz.Chatbot/Commands/TrainHatCommand.cs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
using Fritz.StreamLib.Core;
2+
using System;
3+
using System.Collections.Generic;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
7+
namespace Fritz.Chatbot.Commands
8+
{
9+
public class TrainHatCommand : IBasicCommand2
10+
{
11+
private readonly ITrainHat _TrainHat;
12+
13+
public string Trigger => "trainhat";
14+
public string Description => "Moderators can capture 15 screenshots in an effort to help train the hat detection AI";
15+
public TimeSpan? Cooldown => TimeSpan.FromMinutes(15);
16+
17+
public TrainHatCommand(ITrainHat trainHat)
18+
{
19+
_TrainHat = trainHat;
20+
}
21+
22+
public async Task Execute(IChatService chatService, string userName, bool isModerator, bool isVip, bool isBroadcaster, ReadOnlyMemory<char> rhs)
23+
{
24+
25+
if (!(isModerator || isBroadcaster)) return;
26+
27+
_TrainHat.StartTraining();
28+
await chatService.SendMessageAsync("Started taking screenshots, 1 per minute for the next 15 minutes");
29+
30+
}
31+
32+
public Task Execute(IChatService chatService, string userName, ReadOnlyMemory<char> rhs)
33+
{
34+
throw new NotImplementedException();
35+
}
36+
}
37+
}

‎Fritz.Chatbot/ITrainHat.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using System.Threading.Tasks;
2+
3+
namespace Fritz.Chatbot
4+
{
5+
6+
public interface ITrainHat
7+
{
8+
9+
void StartTraining();
10+
11+
Task AddScreenshot();
12+
13+
}
14+
15+
}

‎Fritz.Chatbot/ScreenshotTrainingService.cs‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training;
2+
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training.Models;
3+
using Microsoft.Extensions.Configuration;
4+
using Microsoft.Extensions.Hosting;
5+
using Microsoft.Extensions.Logging;
6+
using System;
7+
using System.Collections.Generic;
8+
using System.ComponentModel.DataAnnotations;
9+
using System.Text;
10+
using System.Threading;
11+
using System.Threading.Tasks;
12+
13+
namespace Fritz.Chatbot
14+
{
15+
public class ScreenshotTrainingService : IHostedService, ITrainHat
16+
{
17+
18+
// TODO: Track how many images are loaded -- 5k is the maximum for the FREE service
19+
private string _CustomVisionKey = "";
20+
private string _AzureEndpoint = "";
21+
private string _TwitchChannel = "";
22+
private Guid _AzureProjectId;
23+
private readonly ILogger _Logger;
24+
private CancellationTokenSource _TokenSource;
25+
26+
private bool _CurrentlyTraining = false;
27+
private byte _TrainingCount = 0;
28+
private Task _TrainingTask;
29+
private byte _RetryCount = 0;
30+
31+
public string TwitchScreenshotUrl => $"https://static-cdn.jtvnw.net/previews-ttv/live_user_{_TwitchChannel}-1280x720.jpg?_=";
32+
33+
public ScreenshotTrainingService(IConfiguration configuration, ILoggerFactory loggerFactory)
34+
{
35+
36+
_CustomVisionKey = configuration["AzureServices:HatDetection:Key"];
37+
_AzureEndpoint = configuration["AzureServices:HatDetection:CustomVisionEndpoint"];
38+
_TwitchChannel = configuration["StreamServices:Twitch:Channel"];
39+
_AzureProjectId = Guid.Parse(configuration["AzureServices:HatDetection:ProjectId"]);
40+
_Logger = loggerFactory.CreateLogger("ScreenshotTraining");
41+
42+
}
43+
44+
public Task StartAsync(CancellationToken cancellationToken)
45+
{
46+
47+
_TokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
48+
49+
_TrainingTask = Train(_TokenSource.Token);
50+
return Task.CompletedTask;
51+
52+
}
53+
54+
public async Task StopAsync(CancellationToken cancellationToken)
55+
{
56+
57+
_TokenSource?.Cancel();
58+
await _TrainingTask;
59+
60+
}
61+
62+
private async Task Train(CancellationToken token)
63+
{
64+
65+
while (!token.IsCancellationRequested)
66+
{
67+
68+
if (_CurrentlyTraining && _TrainingCount == 15)
69+
{
70+
_CurrentlyTraining = false;
71+
_Logger.LogTrace("Completed screenshot training");
72+
73+
}
74+
else if (_CurrentlyTraining)
75+
{
76+
77+
await AddScreenshot(true);
78+
await Task.Delay(TimeSpan.FromMinutes(1));
79+
80+
}
81+
else
82+
{
83+
84+
await Task.Delay(100);
85+
86+
}
87+
88+
}
89+
90+
}
91+
92+
public void StartTraining()
93+
{
94+
95+
if (_CurrentlyTraining) return;
96+
97+
_Logger.LogTrace("Starting screenshot training");
98+
_TrainingCount = 0;
99+
_CurrentlyTraining = true;
100+
101+
}
102+
103+
private async Task AddScreenshot(bool @internal)
104+
{
105+
106+
if (!@internal && _CurrentlyTraining) return;
107+
108+
try
109+
{
110+
var trainingClient = new CustomVisionTrainingClient()
111+
{
112+
ApiKey = _CustomVisionKey,
113+
Endpoint = _AzureEndpoint
114+
};
115+
116+
var result = await trainingClient.CreateImagesFromUrlsAsync(_AzureProjectId, new ImageUrlCreateBatch(
117+
new List<ImageUrlCreateEntry> {
118+
new ImageUrlCreateEntry(TwitchScreenshotUrl + Guid.NewGuid().ToString())
119+
}
120+
));
121+
122+
if (!result.IsBatchSuccessful && _RetryCount < 3) {
123+
_Logger.LogWarning($"Error while adding screenshot #{_TrainingCount} - trying again in 10 seconds");
124+
await Task.Delay(TimeSpan.FromSeconds(10));
125+
_RetryCount++;
126+
await AddScreenshot(true);
127+
return;
128+
} else if (_RetryCount >= 3) {
129+
130+
_Logger.LogError("Unable to add screenshot to Azure Custom Vision service");
131+
_RetryCount = 0;
132+
return;
133+
134+
}
135+
136+
_RetryCount = 0;
137+
138+
if (_CurrentlyTraining)
139+
{
140+
_TrainingCount++;
141+
_Logger.LogTrace($"Successfully added screenshot #{_TrainingCount}");
142+
}
143+
144+
}
145+
catch (Exception ex)
146+
{
147+
_Logger.LogError($"Error while adding screenshot: {ex.Message}");
148+
}
149+
150+
}
151+
152+
153+
public Task AddScreenshot()
154+
{
155+
156+
return AddScreenshot(false);
157+
158+
}
159+
160+
}
161+
}

‎Fritz.StreamTools/StartupServices/ConfigureServices.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ public static void Execute(IServiceCollection services, IConfiguration configura
6161
RegisterConfiguredServices(services, configuration);
6262
RegisterGitHubServices(services, configuration);
6363

64+
RegisterBotAiServices(services, configuration);
65+
6466

6567
}
6668

@@ -199,6 +201,18 @@ private static void RegisterTwitchPubSub(this IServiceCollection services) {
199201

200202
}
201203

204+
private static void RegisterBotAiServices(IServiceCollection services, IConfiguration configuration)
205+
{
206+
207+
var provider = services.BuildServiceProvider(); // Build a 'temporary' instance of the DI container
208+
var loggerFactory = provider.GetService<ILoggerFactory>();
209+
210+
var service = new ScreenshotTrainingService(configuration, loggerFactory);
211+
services.AddSingleton(service as IHostedService);
212+
services.AddSingleton(service as ITrainHat);
213+
214+
}
215+
202216
private static bool IsTwitchEnabled {
203217
get { return string.IsNullOrEmpty(_Configuration["StreamServices:Twitch:ClientId"]); }
204218
}

0 commit comments

Comments
 (0)