Skip to content

Commit bea9ddd

Browse files
csharpfritzcsharpfritz
committed
Added the Hat Detection Display widget
1 parent 8b08aae commit bea9ddd

File tree

6 files changed

+156
-15
lines changed

6 files changed

+156
-15
lines changed

‎Fritz.Chatbot/Commands/PredictHatCommand.cs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
using System.Threading.Tasks;
1111
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training;
1212
using System.Net;
13+
using Microsoft.AspNetCore.SignalR;
14+
using Fritz.StreamTools.Hubs;
1315

1416
namespace Fritz.Chatbot.Commands
1517
{
@@ -27,15 +29,17 @@ public class PredictHatCommand : IBasicCommand
2729
internal static string IterationName = "";
2830
private ScreenshotTrainingService _TrainHat;
2931
private readonly HatDescriptionRepository _Repository;
32+
private readonly IHubContext<ObsHub> _HubContext;
3033

31-
public PredictHatCommand(IConfiguration configuration, ScreenshotTrainingService service, HatDescriptionRepository repository)
34+
public PredictHatCommand(IConfiguration configuration, ScreenshotTrainingService service, HatDescriptionRepository repository, IHubContext<ObsHub> hubContext)
3235
{
3336
_CustomVisionKey = configuration["AzureServices:HatDetection:Key"];
3437
_AzureEndpoint = configuration["AzureServices:HatDetection:CustomVisionEndpoint"];
3538
_TwitchChannel = configuration["StreamServices:Twitch:Channel"];
3639
_AzureProjectId = Guid.Parse(configuration["AzureServices:HatDetection:ProjectId"]);
3740
_TrainHat = service;
3841
_Repository = repository;
42+
_HubContext = hubContext;
3943
}
4044

4145
public async Task Execute(IChatService chatService, string userName, ReadOnlyMemory<char> rhs)
@@ -51,6 +55,7 @@ public async Task Execute(IChatService chatService, string userName, ReadOnlyMem
5155
Endpoint = _AzureEndpoint,
5256
};
5357

58+
await _HubContext.Clients.All.SendAsync("shutter");
5459
var obsImage = await _TrainHat.GetScreenshotFromObs();
5560

5661
////////////////////////////
@@ -83,9 +88,12 @@ public async Task Execute(IChatService chatService, string userName, ReadOnlyMem
8388
return;
8489
}
8590

86-
await chatService.SendMessageAsync($"csharpClip I think (with {bestMatch.Probability.ToString("0.0%")} certainty) Jeff is currently wearing his {bestMatch.TagName} hat csharpClip");
87-
var desc = await _Repository.GetDescription(bestMatch.TagName);
88-
if (!string.IsNullOrEmpty(desc)) await chatService.SendMessageAsync(desc);
91+
var hatData = (await _Repository.GetHatData(bestMatch.TagName));
92+
var nameToReport = (hatData == null ? bestMatch.TagName : (string.IsNullOrEmpty(hatData.Name) ? bestMatch.TagName : hatData.Name));
93+
await chatService.SendMessageAsync($"csharpClip I think (with {bestMatch.Probability.ToString("0.0%")} certainty) Jeff is currently wearing his {nameToReport} hat csharpClip");
94+
if (hatData != null && !string.IsNullOrEmpty(hatData.Description)) await chatService.SendMessageAsync(hatData.Description);
95+
96+
await _HubContext.Clients.All.SendAsync("hatDetected", bestMatch.Probability.ToString("0.0%"), bestMatch.TagName, nameToReport, hatData?.Description);
8997

9098
}
9199

‎Fritz.Chatbot/Commands/TrainHatCommand.cs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
using Fritz.StreamLib.Core;
2+
using Fritz.StreamTools.Hubs;
3+
using Microsoft.AspNetCore.SignalR;
24
using System;
35
using System.Collections.Generic;
46
using System.Text;
@@ -10,23 +12,28 @@ namespace Fritz.Chatbot.Commands
1012
public class TrainHatCommand : IBasicCommand2
1113
{
1214
private readonly ITrainHat _TrainHat;
15+
private readonly IHubContext<ObsHub> _HubContext;
1316

1417
public string Trigger => "trainhat";
1518
public string Description => "Moderators can capture 15 screenshots in an effort to help train the hat detection AI";
1619
public TimeSpan? Cooldown => TimeSpan.FromMinutes(15);
1720

18-
public TrainHatCommand(ScreenshotTrainingService service)
21+
public TrainHatCommand(ScreenshotTrainingService service, IHubContext<ObsHub> hubContext)
1922
{
2023
_TrainHat = service;
24+
_HubContext = hubContext;
2125
}
2226

2327
public async Task Execute(IChatService chatService, string userName, bool isModerator, bool isVip, bool isBroadcaster, ReadOnlyMemory<char> rhs)
2428
{
2529

2630
if (!(isModerator || isBroadcaster)) return;
2731

28-
_TrainHat.StartTraining();
29-
await chatService.SendMessageAsync($"Started taking screenshots, 1 every {ScreenshotTrainingService.TrainingIntervalInSeconds} seconds for the next {ScreenshotTrainingService.TrainingIntervalInSeconds * ScreenshotTrainingService.TrainingCount} seconds");
32+
var picCount = ScreenshotTrainingService.DefaultTrainingCount;
33+
int.TryParse(rhs.ToString(), out picCount);
34+
_TrainHat.StartTraining(picCount);
35+
await _HubContext.Clients.All.SendAsync("shutter");
36+
await chatService.SendMessageAsync($"Started taking screenshots, 1 every {ScreenshotTrainingService.TrainingIntervalInSeconds} seconds for the next {ScreenshotTrainingService.TrainingIntervalInSeconds * picCount} seconds");
3037

3138
}
3239

‎Fritz.Chatbot/HatDescriptionRepository.cs‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,34 @@ public HatDescriptionRepository(IConfiguration configuration)
3030
/// </summary>
3131
/// <param name="tag">The unique tag for the hat</param>
3232
/// <returns>Description (if any) for the hat</returns>
33-
public async Task<string> GetDescription(string tag) {
33+
public async Task<HatData> GetHatData(string tag) {
3434

3535
try
3636
{
3737
var singleMatch = await _Client.Query(Get(Match(Index("hats_tag_desc"), tag)));
3838

39-
return singleMatch.Get(Field.At("data")).At("description").To<string>().Value;
39+
var hatData = singleMatch.Get(Field.At("data")).To<HatData>().Value;
40+
return hatData;
4041
} catch {
4142
// No result found
42-
return "";
43+
return null;
4344
}
4445
}
4546

47+
48+
public class HatData {
49+
50+
[FaunaField("tag")]
51+
public string Tag { get; set; }
52+
53+
[FaunaField("description")]
54+
public string Description { get; set; }
55+
56+
[FaunaField("name")]
57+
public string? Name { get; set; }
58+
59+
}
60+
4661
}
62+
4763
}

‎Fritz.Chatbot/ITrainHat.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ namespace Fritz.Chatbot
66
public interface ITrainHat
77
{
88

9-
void StartTraining();
9+
void StartTraining(int? count);
1010

1111
Task AddScreenshot();
1212

‎Fritz.Chatbot/ScreenshotTrainingService.cs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ namespace Fritz.Chatbot
2121
{
2222
public class ScreenshotTrainingService : IHostedService, ITrainHat
2323
{
24-
public const int TrainingCount = 15;
24+
public const int DefaultTrainingCount = 15;
25+
private int _TrainingPicCount = 15;
2526
public const int TrainingIntervalInSeconds = 2;
2627

2728
// TODO: Track how many images are loaded -- 5k is the maximum for the FREE service
@@ -75,7 +76,7 @@ private async Task Train(CancellationToken token)
7576
while (!token.IsCancellationRequested)
7677
{
7778

78-
if (_CurrentlyTraining && _TotalPictures == TrainingCount)
79+
if (_CurrentlyTraining && _TotalPictures == DefaultTrainingCount)
7980
{
8081
_CurrentlyTraining = false;
8182
_Logger.LogTrace("Completed screenshot training");
@@ -131,19 +132,20 @@ private async Task UploadCachedScreenshots()
131132
});
132133

133134
_TotalPictures -= (byte)result.Body.Images.Where(r => r.Status != "OK").Count();
134-
if (_TotalPictures >= TrainingCount) _CurrentlyTraining = true;
135+
if (_TotalPictures >= DefaultTrainingCount) _CurrentlyTraining = true;
135136

136137
Console.WriteLine(result.ToString());
137138

138139
}
139140

140-
public void StartTraining()
141+
public void StartTraining(int? count)
141142
{
142143

143144
if (_CurrentlyTraining) return;
144145

145146
_Logger.LogTrace("Starting screenshot training");
146147
_TrainingCount = 0;
148+
_TrainingPicCount = count ?? DefaultTrainingCount;
147149
_CurrentlyTraining = true;
148150

149151
}

‎Fritz.StreamTools/Pages/HatDisplay.cshtml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
@page
2+
@{
3+
}
4+
<!DOCTYPE html>
5+
6+
<html>
7+
<head>
8+
<meta name="viewport" content="width=device-width" />
9+
<title>ToDo Widget</title>
10+
<link rel="stylesheet" href="~/css/site.css" />
11+
<style>
12+
13+
@@keyframes fadeIn {
14+
0% {
15+
opacity: 0;
16+
}
17+
18+
100% {
19+
opacity: 1;
20+
}
21+
}
22+
23+
@@keyframes fadeOut {
24+
0% {
25+
opacity: 1;
26+
}
27+
28+
100% {
29+
opacity: 0;
30+
}
31+
}
32+
33+
.fadeIn {
34+
animation-name: fadeIn;
35+
animation-duration: 1s;
36+
animation-fill-mode: both;
37+
}
38+
39+
.fadeOut {
40+
animation-name: fadeOut;
41+
animation-duration: 1s;
42+
animation-fill-mode: both;
43+
} </style>
44+
<script src="https://kit.fontawesome.com/8ac2e0bf60.js" crossorigin="anonymous"></script>
45+
</head>
46+
<body>
47+
48+
<div id="d" class="todoWidget fadeOut" style="padding:10px">
49+
<h4>Jeff is wearing his <span id="hatName"></span> hat. (<span id="confidence"></span> accuracy)</h4>
50+
<hr style="width: 200px" />
51+
<h4 id="description" style="text-align: left;"></h4>
52+
</div>
53+
54+
<script src="~/lib/signalr/signalr-client.js"></script>
55+
<script>
56+
57+
var debug = false;
58+
var detected = new Audio('@Url.Content(@"~/contents/hatDetected.mp3")');
59+
var shutter = new Audio('@Url.Content(@"~/contents/shutter.mp3")');
60+
var widget = document.getElementById("d");
61+
62+
(function () {
63+
64+
this._hub = new signalR.HubConnectionBuilder()
65+
.withUrl("/obshub")
66+
.withHubProtocol(new signalR.protocols.msgpack.MessagePackHubProtocol())
67+
.build();
68+
69+
this._hub.onclose(() => {
70+
if (debug) console.debug("hub connection closed");
71+
72+
// Hub connection was closed for some reason
73+
let interval = setInterval(() => {
74+
// Try to reconnect hub every 5 secs
75+
this.start(groups).then(() => {
76+
// Reconnect succeeded
77+
clearInterval(interval);
78+
if (this.debug) console.debug("hub reconnected");
79+
});
80+
}, 5000);
81+
});
82+
83+
this._hub.on("shutter", () => {
84+
shutter.play();
85+
});
86+
87+
this._hub.on("hatDetected", (pct, tag, name, desc) => {
88+
if (debug) console.debug(`Hat detected: ${pct} ${tag} ${name} ${desc}`, null);
89+
document.getElementById("hatName").innerText = name;
90+
document.getElementById("confidence").innerText = pct;
91+
document.getElementById("description").innerText = desc;
92+
widget.classList.remove("fadeOut");
93+
widget.classList.add("fadeIn");
94+
detected.play();
95+
96+
window.setTimeout(function () {
97+
widget.classList.remove("fadeIn");
98+
widget.classList.add("fadeOut");
99+
}, 15000);
100+
101+
});
102+
103+
this._hub.start();
104+
105+
})();
106+
</script>
107+
</body>
108+
</html>

0 commit comments

Comments
 (0)