Skip to content

Commit e4ae39c

Browse files
csharpfritzcsharpfritz
authored
Feature obs screenshot (#414)
* Began work on the OBS connection * Starting two-way SignalR connection * Starting to simplify and optimize with Brady * Completed integration with OBS * Started making OBS Client more testable so that we can isolate pictures without being on stream * Now saing test screenshots from OBS * Now properly cropping and formatting screenshots directly from OBS * Added logging for the unauthorized queries * Fix #413 - Now fetching app access token based on ClientId and Client Secret * Fixed connections to ObsProxy and connected the predict hat command
1 parent a97ecd3 commit e4ae39c

23 files changed

+672
-64
lines changed

‎Fritz.Chatbot/AttentionHub.cs‎

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@
77

88
namespace Fritz.StreamTools.Hubs
99
{
10-
public interface IAttentionHubClient
11-
{
10+
public interface IAttentionHubClient
11+
{
1212

13-
// Cheer 200 parithon 12/18/2018
14-
// Cheer 500 pharewings 12/18/2018
13+
// Cheer 200 parithon 12/18/2018
14+
// Cheer 500 pharewings 12/18/2018
1515
Task AlertFritz();
1616
Task ClientConnected(string connectionId);
1717
Task SummonScott();
@@ -20,33 +20,34 @@ public interface IAttentionHubClient
2020

2121
Task NotifyChannelPoints(ChannelPointRedemption redemption);
2222

23-
}
24-
25-
public class AttentionHub : Hub<IAttentionHubClient>, IAttentionClient
26-
{
27-
public override Task OnConnectedAsync()
28-
{
29-
return this.Clients.Others.ClientConnected(this.Context.ConnectionId);
3023
}
3124

32-
public Task AlertFritz()
25+
public class AttentionHub : Hub<IAttentionHubClient>, IAttentionClient
3326
{
34-
return this.Clients.Others.AlertFritz();
35-
}
27+
public override Task OnConnectedAsync()
28+
{
29+
return this.Clients.Others.ClientConnected(this.Context.ConnectionId);
30+
}
3631

37-
public Task SummonScott()
38-
{
32+
public Task AlertFritz()
33+
{
34+
return this.Clients.Others.AlertFritz();
35+
}
3936

40-
return this.Clients.Others.SummonScott();
37+
public Task SummonScott()
38+
{
4139

42-
}
40+
return this.Clients.Others.SummonScott();
4341

44-
public Task PlaySoundEffect(string fileName)
45-
{
42+
}
43+
44+
public Task PlaySoundEffect(string fileName)
45+
{
46+
47+
return this.Clients.Others.PlaySoundEffect(fileName);
4648

47-
return this.Clients.Others.PlaySoundEffect(fileName);
49+
}
4850

4951
}
5052

51-
}
5253
}

‎Fritz.Chatbot/Commands/AddHatCommand.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,16 @@ namespace Fritz.Chatbot.Commands
1212
{
1313
public class AddHatCommand : IBasicCommand2
1414
{
15-
private readonly ITrainHat _TrainHat;
15+
16+
private readonly ScreenshotTrainingService _TrainHat;
1617

1718
public string Trigger => "addhat";
1819
public string Description => "Moderators can add a screenshot of the stream to the image detection library";
1920
public TimeSpan? Cooldown => TimeSpan.FromMinutes(2);
2021

21-
public AddHatCommand(ITrainHat trainHat)
22+
public AddHatCommand(ScreenshotTrainingService service)
2223
{
23-
_TrainHat = trainHat;
24+
_TrainHat = service;
2425
}
2526

2627
public async Task Execute(IChatService chatService, string userName, bool isModerator, bool isVip, bool isBroadcaster, ReadOnlyMemory<char> rhs)

‎Fritz.Chatbot/Commands/PredictHatCommand.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,15 @@ public class PredictHatCommand : IBasicCommand
2525
private Guid _AzureProjectId;
2626

2727
private static string _IterationName = "";
28+
private ScreenshotTrainingService _TrainHat;
2829

29-
public PredictHatCommand(IConfiguration configuration)
30+
public PredictHatCommand(IConfiguration configuration, ScreenshotTrainingService service)
3031
{
3132
_CustomVisionKey = configuration["AzureServices:HatDetection:Key"];
3233
_AzureEndpoint = configuration["AzureServices:HatDetection:CustomVisionEndpoint"];
3334
_TwitchChannel = configuration["StreamServices:Twitch:Channel"];
3435
_AzureProjectId = Guid.Parse(configuration["AzureServices:HatDetection:ProjectId"]);
36+
_TrainHat = service;
3537
}
3638

3739
public string TwitchScreenshotUrl => $"https://static-cdn.jtvnw.net/previews-ttv/live_user_{_TwitchChannel}-1280x720.jpg?_=";
@@ -50,11 +52,14 @@ public async Task Execute(IChatService chatService, string userName, ReadOnlyMem
5052
Endpoint = _AzureEndpoint
5153
};
5254

55+
var obsImage = await _TrainHat.GetScreenshotFromObs();
56+
57+
////////////////////////////
5358

5459
ImagePrediction result;
5560
try
5661
{
57-
result = await client.DetectImageUrlWithNoStoreAsync(_AzureProjectId, _IterationName, new ImageUrl(TwitchScreenshotUrl));
62+
result = await client.DetectImageWithNoStoreAsync(_AzureProjectId, _IterationName, obsImage);
5863
} catch (CustomVisionErrorException ex) {
5964

6065

‎Fritz.Chatbot/Commands/ShoutoutCommand.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,12 @@ public class ShoutoutCommand : IBasicCommand2
1212
private readonly HttpClient _HttpClient;
1313
private readonly ILogger _Logger;
1414

15-
public ShoutoutCommand(IHttpClientFactory httpClientFactory, ILoggerFactory logger)
15+
public ShoutoutCommand(IHttpClientFactory httpClientFactory, ILoggerFactory logger, TwitchTokenConfig twitchConfig = null)
1616
{
1717

1818
_HttpClient = httpClientFactory.CreateClient("ShoutoutCommand");
1919
_HttpClient.BaseAddress = new Uri("https://api.twitch.tv/helix/users");
20+
_HttpClient.DefaultRequestHeaders.Add("Authorization", $"Bearer {TwitchTokenConfig.Tokens.access_token}");
2021

2122
_Logger = logger.CreateLogger(nameof(ShoutoutCommand));
2223

@@ -41,7 +42,11 @@ public async Task Execute(IChatService chatService, string userName, bool isMode
4142

4243
rhsTest = WebUtility.UrlEncode(rhsTest);
4344
var result = await _HttpClient.GetAsync($"?login={rhsTest}");
44-
if (result.StatusCode != HttpStatusCode.OK)
45+
if (result.StatusCode == HttpStatusCode.Unauthorized) {
46+
_Logger.LogError("Request to Twitch endpoint was unauthorized -- consider rotating keys");
47+
return;
48+
}
49+
else if (result.StatusCode != HttpStatusCode.OK)
4550
{
4651
_Logger.LogWarning($"Unable to verify Shoutout for {rhsTest}");
4752
return;

‎Fritz.Chatbot/Commands/TrainHatCommand.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Collections.Generic;
44
using System.Text;
55
using System.Threading.Tasks;
6+
using System.Transactions;
67

78
namespace Fritz.Chatbot.Commands
89
{
@@ -14,9 +15,9 @@ public class TrainHatCommand : IBasicCommand2
1415
public string Description => "Moderators can capture 15 screenshots in an effort to help train the hat detection AI";
1516
public TimeSpan? Cooldown => TimeSpan.FromMinutes(15);
1617

17-
public TrainHatCommand(ITrainHat trainHat)
18+
public TrainHatCommand(ScreenshotTrainingService service)
1819
{
19-
_TrainHat = trainHat;
20+
_TrainHat = service;
2021
}
2122

2223
public async Task Execute(IChatService chatService, string userName, bool isModerator, bool isVip, bool isBroadcaster, ReadOnlyMemory<char> rhs)

‎Fritz.Chatbot/Fritz.Chatbot.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="2.2.0" />
2121
<PackageReference Include="Microsoft.Extensions.Logging" Version="2.1.1" />
2222
<PackageReference Include="NetCoreAudio" Version="1.5.0" />
23-
<PackageReference Include="Newtonsoft.Json" Version="12.0.2" />
23+
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
2424
</ItemGroup>
2525

2626
<ItemGroup>

‎Fritz.Chatbot/FritzBot.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
using Fritz.Chatbot.Commands;
22
using Fritz.StreamLib.Core;
3+
using Fritz.StreamTools.Hubs;
4+
using Microsoft.AspNetCore.SignalR;
35
using Microsoft.Extensions.Configuration;
46
using Microsoft.Extensions.DependencyInjection;
57
using Microsoft.Extensions.Hosting;
@@ -30,7 +32,7 @@ public class FritzBot : IHostedService
3032

3133
public TimeSpan CooldownTime { get; }
3234

33-
public FritzBot(IConfiguration configuration, IServiceProvider serviceProvider, ILoggerFactory loggerFactory = null)
35+
public FritzBot(IConfiguration configuration, IServiceProvider serviceProvider, ILoggerFactory loggerFactory = null, IHubContext<ObsHub, ITakeScreenshots> hubContext = null)
3436
{
3537
if (configuration == null)
3638
{
@@ -55,6 +57,7 @@ public FritzBot(IConfiguration configuration, IServiceProvider serviceProvider,
5557
_OtherBots = String.IsNullOrEmpty(configuration[$"{ConfigurationRoot}:Otherbots"]) ? new[] { "nightbot","fritzbot","streamelements","pretzelrocks" } : configuration[$"{ConfigurationRoot}:Otherbots"].Split(',');
5658

5759
_logger?.LogInformation("Command cooldown set to {0}", CooldownTime);
60+
5861
}
5962

6063
/// <summary>

‎Fritz.Chatbot/ObsHub.cs‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
using Fritz.StreamLib.Core;
2+
using Microsoft.AspNetCore.SignalR;
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Diagnostics;
6+
using System.IO;
7+
using System.Text;
8+
using System.Threading.Channels;
9+
using System.Threading.Tasks;
10+
11+
namespace Fritz.StreamTools.Hubs
12+
{
13+
14+
public class ObsHub : Hub<ITakeScreenshots> {
15+
16+
public static int ConnectedCount = 0;
17+
public static List<string> _ConnectionIds = new List<string>();
18+
19+
public override async Task OnConnectedAsync()
20+
{
21+
22+
_ConnectionIds.Add(Context.ConnectionId);
23+
24+
ConnectedCount++;
25+
26+
await base.OnConnectedAsync();
27+
28+
}
29+
30+
public override async Task OnDisconnectedAsync(Exception exception)
31+
{
32+
ConnectedCount--;
33+
34+
await base.OnDisconnectedAsync(exception);
35+
}
36+
37+
public async Task PostScreenshot(IAsyncEnumerable<string> stream) {
38+
39+
var sb = new StringBuilder();
40+
await foreach (var item in stream)
41+
{
42+
sb.Append(item);
43+
}
44+
45+
Debug.WriteLine(sb.Length);
46+
var cleanString = sb.ToString().Replace("data:image/png;base64,", "");
47+
48+
var bytes = Convert.FromBase64String(cleanString);
49+
50+
ScreenshotSink.Instance.OnScreenshotReceived(bytes);
51+
52+
}
53+
54+
}
55+
56+
public interface IServerTakeScreenshot
57+
{
58+
Task TakeScreenshot();
59+
}
60+
61+
public class ScreenshotReceivedEventArgs : EventArgs {
62+
63+
public Stream Screenshot { get; set; }
64+
65+
}
66+
67+
public class ScreenshotSink {
68+
69+
public static readonly ScreenshotSink Instance = new ScreenshotSink();
70+
71+
private ScreenshotSink() { }
72+
73+
public event EventHandler<ScreenshotReceivedEventArgs> ScreenshotReceived;
74+
75+
public void OnScreenshotReceived(byte[] imageData) {
76+
77+
var args = new ScreenshotReceivedEventArgs() { Screenshot = new MemoryStream(imageData) };
78+
ScreenshotReceived?.Invoke(null, args);
79+
80+
}
81+
82+
}
83+
84+
}

‎Fritz.Chatbot/ScreenshotTrainingService.cs‎

Lines changed: 45 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,17 @@
1-
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training;
1+
using Fritz.StreamLib.Core;
2+
using Fritz.StreamTools.Hubs;
3+
using Microsoft.AspNetCore.SignalR;
4+
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training;
25
using Microsoft.Azure.CognitiveServices.Vision.CustomVision.Training.Models;
36
using Microsoft.Extensions.Configuration;
7+
using Microsoft.Extensions.DependencyInjection;
48
using Microsoft.Extensions.Hosting;
59
using Microsoft.Extensions.Logging;
610
using System;
711
using System.Collections.Generic;
812
using System.ComponentModel.DataAnnotations;
13+
using System.IO;
14+
using System.Linq;
915
using System.Text;
1016
using System.Threading;
1117
using System.Threading.Tasks;
@@ -18,29 +24,27 @@ public class ScreenshotTrainingService : IHostedService, ITrainHat
1824
// TODO: Track how many images are loaded -- 5k is the maximum for the FREE service
1925
private string _CustomVisionKey = "";
2026
private string _AzureEndpoint = "";
21-
private string _TwitchChannel = "";
2227
private Guid _AzureProjectId;
23-
private readonly ILogger _Logger;
28+
private ILogger _Logger;
29+
private IServiceProvider _Services;
2430
private CancellationTokenSource _TokenSource;
2531

2632
private bool _CurrentlyTraining = false;
2733
private byte _TrainingCount = 0;
2834
private Task _TrainingTask;
2935
private byte _RetryCount = 0;
3036

31-
public string TwitchScreenshotUrl => $"https://static-cdn.jtvnw.net/previews-ttv/live_user_{_TwitchChannel}-1280x720.jpg?_=";
32-
33-
public ScreenshotTrainingService(IConfiguration configuration, ILoggerFactory loggerFactory)
37+
public ScreenshotTrainingService(IConfiguration configuration, ILoggerFactory loggerFactory, IServiceProvider services)
3438
{
3539

3640
_CustomVisionKey = configuration["AzureServices:HatDetection:Key"];
3741
_AzureEndpoint = configuration["AzureServices:HatDetection:CustomVisionEndpoint"];
38-
_TwitchChannel = configuration["StreamServices:Twitch:Channel"];
3942
_AzureProjectId = Guid.Parse(configuration["AzureServices:HatDetection:ProjectId"]);
4043
_Logger = loggerFactory.CreateLogger("ScreenshotTraining");
41-
44+
_Services = services;
4245
}
4346

47+
4448
public Task StartAsync(CancellationToken cancellationToken)
4549
{
4650

@@ -113,11 +117,12 @@ private async Task AddScreenshot(bool @internal)
113117
Endpoint = _AzureEndpoint
114118
};
115119

116-
var result = await trainingClient.CreateImagesFromUrlsAsync(_AzureProjectId, new ImageUrlCreateBatch(
117-
new List<ImageUrlCreateEntry> {
118-
new ImageUrlCreateEntry(TwitchScreenshotUrl + Guid.NewGuid().ToString())
119-
}
120-
));
120+
var imageStream = await GetScreenshotFromObs();
121+
// TODO: If imageStream is null, handle gracefully
122+
123+
var result = await trainingClient.CreateImagesFromDataAsync(_AzureProjectId,
124+
imageStream
125+
);
121126

122127
if (!result.IsBatchSuccessful && _RetryCount < 3) {
123128
_Logger.LogWarning($"Error while adding screenshot #{_TrainingCount} - trying again in 10 seconds");
@@ -144,11 +149,37 @@ private async Task AddScreenshot(bool @internal)
144149
}
145150
catch (Exception ex)
146151
{
147-
_Logger.LogError($"Error while adding screenshot: {ex.Message}");
152+
153+
_Logger.LogError($"Error while adding screenshot: {ex.Message}");
148154
}
149155

150156
}
151157

158+
internal async Task<Stream> GetScreenshotFromObs()
159+
{
160+
161+
Stream result = null;
162+
163+
ScreenshotSink.Instance.ScreenshotReceived += (obj, args) =>
164+
{
165+
result = args.Screenshot;
166+
};
167+
168+
using (var scope = _Services.CreateScope())
169+
{
170+
var obsContext = scope.ServiceProvider.GetRequiredService<IHubContext<ObsHub, ITakeScreenshots>>();
171+
await obsContext.Clients.All.TakeScreenshot();
172+
}
173+
var i = 0;
174+
while (result == null) {
175+
await Task.Delay(100);
176+
i++;
177+
if (i >= 100) break;
178+
}
179+
180+
return result;
181+
182+
}
152183

153184
public Task AddScreenshot()
154185
{

0 commit comments

Comments
 (0)