Skip to content

Commit 37c6f23

Browse files
csharpfritzcsharpfritz
committed
Updated imageDescriptor to better detect images
1 parent 12a99dc commit 37c6f23

File tree

4 files changed

+119
-54
lines changed

4 files changed

+119
-54
lines changed

‎Fritz.Chatbot/Commands/ImageDescriptorCommand.cs

Lines changed: 111 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,93 +1,154 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Linq;
34
using System.Net.Http;
45
using System.Text;
56
using System.Text.RegularExpressions;
7+
using System.Threading;
68
using System.Threading.Tasks;
9+
using System.Web;
710
using Fritz.Chatbot.Models;
811
using Fritz.StreamLib.Core;
912
using Microsoft.Extensions.Configuration;
1013
using Newtonsoft.Json;
1114

1215
namespace Fritz.Chatbot.Commands
1316
{
14-
public class ImageDescriptorCommand : IExtendedCommand
17+
public class ImageDescriptorCommand : IExtendedCommand
18+
{
19+
public string Name => "Image";
20+
public string Description => "Inspect images and report to the chat room what they contain using Vision API";
21+
public int Order => 10;
22+
public bool Final => false;
23+
24+
private readonly string _AzureUrl;
25+
private readonly string _AzureApiKey;
26+
private string ImageUrl;
27+
private string v1;
28+
private string v2;
29+
30+
public TimeSpan? Cooldown => null;
31+
32+
private static readonly Regex _UrlCheck = new Regex(@"http(s)?:?(\/\/[^""']*\.(?:png|jpg|jpeg|gif))", RegexOptions.IgnoreCase | RegexOptions.Compiled);
33+
private readonly IHttpClientFactory _ClientFactory;
34+
35+
public ImageDescriptorCommand(IConfiguration config, IHttpClientFactory clientFactory) : this(config["FritzBot:VisionApiBaseUrl"], config["FritzBot:VisionApiKey"])
1536
{
16-
public string Name => "Image";
17-
public string Description => "Inspect images and report to the chat room what they contain using Vision API";
18-
public int Order => 10;
19-
public bool Final => false;
37+
_ClientFactory = clientFactory;
38+
}
2039

21-
private readonly string _AzureUrl;
22-
private readonly string _AzureApiKey;
23-
private string ImageUrl;
24-
private string v1;
25-
private string v2;
40+
public ImageDescriptorCommand(string azureUrl, string azureKey)
41+
{
42+
_AzureUrl = azureUrl;
43+
_AzureApiKey = azureKey;
44+
}
2645

27-
public TimeSpan? Cooldown => null;
46+
public bool CanExecute(string userName, string fullCommandText)
47+
{
2848

29-
private static readonly Regex _UrlCheck = new Regex(@"http(s)?:?(\/\/[^""']*\.(?:png|jpg|jpeg|gif))", RegexOptions.IgnoreCase | RegexOptions.Compiled);
49+
// Match the regular expression pattern against a text string.
50+
var imageCheck = _UrlCheck.Match(fullCommandText);
51+
if (imageCheck.Captures.Count == 0)
52+
return false;
53+
this.ImageUrl = imageCheck.Captures[0].Value;
3054

31-
public ImageDescriptorCommand(IConfiguration config) : this(config["FritzBot:VisionApiBaseUrl"], config["FritzBot:VisionApiKey"])
32-
{
33-
}
55+
return (!ImageUrl.Contains('#') && imageCheck.Captures.Count > 0) && ValidImageType(ImageUrl).GetAwaiter().GetResult();
3456

35-
public ImageDescriptorCommand(string azureUrl, string azureKey)
36-
{
37-
_AzureUrl = azureUrl;
38-
_AzureApiKey = azureKey;
39-
}
57+
async Task<bool> ValidImageType(string url) {
4058

41-
public bool CanExecute(string userName, string fullCommandText)
42-
{
59+
HttpResponseMessage response;
60+
61+
using (var client = _ClientFactory.CreateClient("ImageDescriptor"))
62+
{
63+
try
64+
{
65+
var request = new HttpRequestMessage
66+
{
67+
Method = HttpMethod.Head,
68+
RequestUri = new Uri(url)
69+
};
70+
response = await client.SendAsync(request, cancellationToken: CancellationToken.None);
71+
response.EnsureSuccessStatusCode();
72+
} catch
73+
{
74+
return false;
75+
}
76+
77+
return (response.Content.Headers.ContentType.MediaType.ToLowerInvariant().StartsWith("image/"));
78+
79+
}
4380

44-
// Match the regular expression pattern against a text string.
45-
var imageCheck = _UrlCheck.Match(fullCommandText);
46-
if (imageCheck.Captures.Count == 0)
47-
return false;
48-
this.ImageUrl = imageCheck.Captures[0].Value;
49-
return (imageCheck.Captures.Count > 0);
5081
}
5182

52-
/// param name="fullCommandText" (this is the URL of the image we already found)
53-
public async Task Execute(IChatService chatService, string userName, string fullCommandText)
54-
{
83+
}
5584

56-
// TODO: Pull from ASP.NET Core Dependency Injection
57-
var client = new HttpClient();
85+
/// param name="fullCommandText" (this is the URL of the image we already found)
86+
public async Task Execute(IChatService chatService, string userName, string fullCommandText)
87+
{
88+
89+
// Cheer 100 themichaeljolley 01/3/19
90+
// Cheer 300 electrichavoc 01/3/19
91+
// Cheer 300 devlead 01/3/19
92+
// Cheer 100 brandonsatrom 01/3/19
93+
// Cheer 642 cpayette 01/3/19
94+
// Cheer 500 robertables 01/3/19
95+
// Cheer 100 johanb 01/3/19
96+
// Cheer 1000 bobtabor 01/3/19
97+
98+
var result = string.Empty;
99+
100+
// TODO: Pull from ASP.NET Core Dependency Injection
101+
using (var client = _ClientFactory.CreateClient("ImageDescriptor"))
102+
{
58103
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", _AzureApiKey);
59104

60105
var requestParameters = "visualFeatures=Categories,Description,Color,Adult&language=en";
61106
var uri = _AzureUrl + "?" + requestParameters;
62107

63-
var body = "{\"url\":\"" + ImageUrl + "\"}";
108+
var body = JsonConvert.SerializeObject(new { url = ImageUrl });
64109
var content = new StringContent(body, Encoding.UTF8, "application/json");
65110

66111
var apiResponse = await client.PostAsync(uri, content);
67-
apiResponse.EnsureSuccessStatusCode();
68-
var result = await apiResponse.Content.ReadAsStringAsync();
69-
apiResponse.Dispose();
70-
71-
var visionDescription = JsonConvert.DeserializeObject<VisionDescription>(result);
72112

73-
if (visionDescription.adult.isAdultContent)
113+
try
74114
{
75-
await chatService.SendMessageAsync($"Hey {userName} - we don't like adult content here!");
76-
// TODO: Timeout / Ban user
77-
return;
115+
apiResponse.EnsureSuccessStatusCode();
78116
}
79-
80-
if (visionDescription.adult.isRacyContent)
117+
catch (Exception)
81118
{
82-
await chatService.SendMessageAsync($"Hey {userName} - that's too racy ({visionDescription.adult.racyScore,0:P2}) for our chat room!");
83-
// TODO: Timeout user
119+
await chatService.SendMessageAsync($"Unable to inspect the image from {userName}");
84120
return;
85121
}
122+
result = await apiResponse.Content.ReadAsStringAsync();
123+
apiResponse.Dispose();
124+
}
86125

87-
var description = $"{userName} Photo ({visionDescription.description.captions[0].confidence,0:P2}): {visionDescription.description.captions[0].text}";
126+
var visionDescription = JsonConvert.DeserializeObject<VisionDescription>(result);
88127

89-
await chatService.SendMessageAsync(description);
128+
if (visionDescription.adult.isAdultContent && visionDescription.adult.adultScore > 0.85F)
129+
{
130+
await chatService.SendMessageAsync($"Hey {userName} - we don't like adult content here!");
131+
// TODO: Timeout / Ban user
132+
return;
133+
}
134+
135+
if (visionDescription.adult.isRacyContent)
136+
{
137+
await chatService.SendMessageAsync($"Hey {userName} - that's too racy ({visionDescription.adult.racyScore,0:P2}) for our chat room!");
138+
// TODO: Timeout user
139+
return;
140+
}
141+
142+
if (visionDescription.description.captions.Length == 0 && visionDescription.categories.Length > 0)
143+
{
144+
await chatService.SendMessageAsync($"No caption for the image submitted by {userName}, but it is: '{string.Join(',', visionDescription.categories.Select(c => c.name))}'");
145+
return;
146+
}
147+
148+
var description = $"{userName} Photo ({visionDescription.description.captions[0].confidence,0:P2}): {visionDescription.description.captions[0].text}";
149+
150+
await chatService.SendMessageAsync(description);
90151

91-
}
92152
}
153+
}
93154
}

‎Fritz.Chatbot/FritzBot.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ private async ValueTask<bool> HandleExtendedCommands(IChatService chatService, C
213213

214214
AfterExecute(user, cmd.Name);
215215

216-
return cmd.Final;
216+
if (cmd.Final) break;
217217
}
218218
}
219219

‎Fritz.StreamTools/StartupServices/ConfigureServices.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ private static void RegisterGitHubServices(IServiceCollection services, IConfigu
7676
});
7777

7878
services.AddHttpClient("DiscoverDotNet");
79+
services.AddHttpClient("ImageDescriptor");
7980

8081
services.AddHttpClient("ShoutoutCommand", c =>
8182
{

‎Test/ImageCommand/MessageTest.cs

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,16 +60,19 @@ public async void Identifies_Guitar()
6060
})
6161
.Returns(Task.FromResult(true));
6262

63-
var sut = new ImageDescriptorCommand("testlocation", "testkey");
63+
var sut = new ImageDescriptorCommand(
64+
"testlocation",
65+
"testkey");
6466

65-
// Act
67+
// Act
68+
//var urlToTest = "https://media.discordapp.net/attachments/336580722653528075/550152641632534558/unknown.png?width=1020&height=574";
6669
await sut.Execute(chatService.Object, "test", url);
6770

6871
// Assert
6972
Assert.Contains("guitar", outDescription);
7073

7174
}
7275

76+
}
7377

74-
}
7578
}

0 commit comments

Comments
 (0)