Skip to content

Commit 7c4bac2

Browse files
Enable Application Insights For Non Hosted Scenario (#1649)
## Why make this change? - Closes #1601 - This PR aims to Integrate Application Insights to Data API builder, which will allow developers to monitor and gain insights into the performance and usage of their applications. ## What is this change? - Added a new optional property called `Telemetry` to runtime config that will contain information about ApplicationInsights connection string. users can enable/disable it as well. - If Application Insights is enabled, All the Application Logs that is currently generated by Data API builder will now be sent to Application Insights. ## How was this tested? - [X] Manual Tests: Logs are being populated in the Application Insights portal - [X] Unit Tests ## Images from Azure Portal 1. Live Metrics ![image](https://github.com/Azure/data-api-builder/assets/102276754/990a6d19-7c40-4e89-b8ed-6ff8698421d6) 2. Logs ![image](https://github.com/Azure/data-api-builder/assets/102276754/a882651a-be5f-4c6c-ae15-29a8869ee33d) 3. Requests ![image](https://github.com/Azure/data-api-builder/assets/102276754/715789ef-93a0-48f7-a719-50b1d1966eda) 4. Exceptions ![image](https://github.com/Azure/data-api-builder/assets/102276754/34834e5d-91e5-46f3-aeff-9fb25e249545) --------- Co-authored-by: Sean Leonard <[email protected]>
1 parent 734f1b7 commit 7c4bac2

File tree

12 files changed

+403
-3
lines changed

12 files changed

+403
-3
lines changed

src/Auth/Azure.DataApiBuilder.Auth.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
<ItemGroup>
1616
<PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" />
17+
<PackageReference Include="Microsoft.AspNetCore.Http" />
1718
<PackageReference Include="StyleCop.Analyzers">
1819
<PrivateAssets>all</PrivateAssets>
1920
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Azure.DataApiBuilder.Config.ObjectModel;
5+
6+
/// <summary>
7+
/// Represents the options for configuring Application Insights.
8+
/// </summary>
9+
public record ApplicationInsightsOptions(bool Enabled = false, string? ConnectionString = null)
10+
{ }

src/Config/ObjectModel/RuntimeOptions.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,9 @@
33

44
namespace Azure.DataApiBuilder.Config.ObjectModel;
55

6-
public record RuntimeOptions(RestRuntimeOptions Rest, GraphQLRuntimeOptions GraphQL, HostOptions Host, string? BaseRoute = null);
6+
public record RuntimeOptions(
7+
RestRuntimeOptions Rest,
8+
GraphQLRuntimeOptions GraphQL,
9+
HostOptions Host,
10+
string? BaseRoute = null,
11+
TelemetryOptions? Telemetry = null);
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace Azure.DataApiBuilder.Config.ObjectModel;
5+
6+
/// <summary>
7+
/// Represents the options for telemetry.
8+
/// </summary>
9+
public record TelemetryOptions(ApplicationInsightsOptions? ApplicationInsights)
10+
{ }

src/Directory.Packages.props

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,20 @@
1313
<PackageVersion Include="Humanizer" Version="2.14.1" />
1414
<PackageVersion Include="Humanizer.Core" Version="2.14.1" />
1515
<PackageVersion Include="DotNetEnv" Version="2.5.0" />
16+
<PackageVersion Include="Microsoft.ApplicationInsights" Version="2.21.0" />
17+
<PackageVersion Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
1618
<PackageVersion Include="Microsoft.AspNetCore.Authorization" Version="6.0.14" />
1719
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.14" />
1820
<PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
21+
<PackageVersion Include="Microsoft.AspNetCore.Http" Version="2.2.0" />
1922
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="6.0.14" />
2023
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="6.0.14" />
2124
<PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.20.0" />
2225
<!--When updating Microsoft.Data.SqlClient, update license URL in scripts/notice-generation.ps1-->
2326
<PackageVersion Include="Microsoft.Data.SqlClient" Version="5.1.1" />
2427
<PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />
2528
<PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
29+
<PackageVersion Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.21.0" />
2630
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
2731
<PackageVersion Include="Microsoft.OData.Edm" Version="7.12.5" />
2832
<PackageVersion Include="Microsoft.OData.Core" Version="7.12.5" />

src/Product/ProductInfo.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ public static class ProductInfo
1111
public const string DEFAULT_VERSION = "1.0.0";
1212
public const string DAB_APP_NAME_ENV = "DAB_APP_NAME_ENV";
1313
public static readonly string DEFAULT_APP_NAME = $"dab_oss_{ProductInfo.GetProductVersion()}";
14+
public static readonly string ROLE_NAME = "DataApiBuilder";
1415

1516
/// <summary>
1617
/// Reads the product version from the executing assembly's file version information.

src/Service.Tests/Configuration/ConfigurationTests.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2494,6 +2494,17 @@ public static RuntimeConfig InitMinimalRuntimeConfig(
24942494
{ entityName, entity }
24952495
};
24962496

2497+
// Adding an entity with only Authorized Access
2498+
Entity anotherEntity = new(
2499+
Source: new("publishers", EntitySourceType.Table, null, null),
2500+
Rest: null,
2501+
GraphQL: new(Singular: "publisher", Plural: "publishers"),
2502+
Permissions: new[] { GetMinimalPermissionConfig(AuthorizationResolver.ROLE_AUTHENTICATED) },
2503+
Relationships: null,
2504+
Mappings: null
2505+
);
2506+
entityMap.Add("Publisher", anotherEntity);
2507+
24972508
return new(
24982509
Schema: "IntegrationTestMinimalSchema",
24992510
DataSource: dataSource,
Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System.Collections.Generic;
5+
using System.IO;
6+
using System.Linq;
7+
using System.Net.Http;
8+
using System.Net.Http.Json;
9+
using System.Threading.Tasks;
10+
using Azure.DataApiBuilder.Config.ObjectModel;
11+
using Microsoft.ApplicationInsights.Channel;
12+
using Microsoft.ApplicationInsights.DataContracts;
13+
using Microsoft.AspNetCore.TestHost;
14+
using Microsoft.IdentityModel.Tokens;
15+
using Microsoft.VisualStudio.TestTools.UnitTesting;
16+
using static Azure.DataApiBuilder.Service.Tests.Configuration.ConfigurationTests;
17+
18+
namespace Azure.DataApiBuilder.Service.Tests.Configuration;
19+
20+
/// <summary>
21+
/// Contains tests for telemetry functionality.
22+
/// </summary>
23+
[TestClass, TestCategory(TestCategory.MSSQL)]
24+
public class TelemetryTests
25+
{
26+
public TestContext TestContext { get; set; }
27+
private const string TEST_APP_INSIGHTS_CONN_STRING = "InstrumentationKey=testKey;IngestionEndpoint=https://localhost/;LiveEndpoint=https://localhost/";
28+
29+
private const string CONFIG_WITH_TELEMETRY = "dab-telemetry-test-config.json";
30+
private const string CONFIG_WITHOUT_TELEMETRY = "dab-no-telemetry-test-config.json";
31+
private static RuntimeConfig _configuration;
32+
33+
/// <summary>
34+
/// Creates runtime config file with specified telemetry options.
35+
/// </summary>
36+
/// <param name="configFileName">Name of the config file to be created.</param>
37+
/// <param name="isTelemetryEnabled">Whether telemetry is enabled or not.</param>
38+
/// <param name="telemetryConnectionString">Telemetry connection string.</param>
39+
public static void SetUpTelemetryInConfig(string configFileName, bool isTelemetryEnabled, string telemetryConnectionString)
40+
{
41+
DataSource dataSource = new(DatabaseType.MSSQL,
42+
GetConnectionStringFromEnvironmentConfig(environment: TestCategory.MSSQL), Options: null);
43+
44+
_configuration = InitMinimalRuntimeConfig(dataSource, graphqlOptions: new(), restOptions: new());
45+
46+
TelemetryOptions _testTelemetryOptions = new(new ApplicationInsightsOptions(isTelemetryEnabled, telemetryConnectionString));
47+
_configuration = _configuration with { Runtime = _configuration.Runtime with { Telemetry = _testTelemetryOptions } };
48+
49+
File.WriteAllText(configFileName, _configuration.ToJson());
50+
}
51+
52+
/// <summary>
53+
/// Cleans up the test environment by deleting the runtime config with telemetry options.
54+
/// </summary>
55+
[TestCleanup]
56+
public void CleanUpTelemetryConfig()
57+
{
58+
File.Delete(CONFIG_WITH_TELEMETRY);
59+
File.Delete(CONFIG_WITHOUT_TELEMETRY);
60+
Startup.AppInsightsOptions = new();
61+
Startup.CustomTelemetryChannel = null;
62+
}
63+
64+
/// <summary>
65+
/// Test for non-hosted scenario.
66+
/// Tests that different telemetry items such as Traces or logs, Exceptions and Requests
67+
/// are correctly sent to application Insights when enabled.
68+
/// Also asserting on their respective properties.
69+
/// </summary>
70+
/// <note>
71+
/// Commenting Assert on Request Telemetry as it is flaky, sometimes passing sometimes failing.
72+
/// while on manual testing it is working fine and we see all the request telemetryItems in Application Insights.
73+
/// Issue to track the fix for this test: https://github.com/Azure/data-api-builder/issues/1734
74+
[TestMethod]
75+
public async Task TestTelemetryItemsAreSentCorrectly_NonHostedScenario()
76+
{
77+
SetUpTelemetryInConfig(CONFIG_WITH_TELEMETRY, isTelemetryEnabled: true, TEST_APP_INSIGHTS_CONN_STRING);
78+
79+
string[] args = new[]
80+
{
81+
$"--ConfigFileName={CONFIG_WITH_TELEMETRY}"
82+
};
83+
84+
ITelemetryChannel telemetryChannel = new CustomTelemetryChannel()
85+
{
86+
EndpointAddress = "https://localhost/"
87+
};
88+
Startup.CustomTelemetryChannel = telemetryChannel;
89+
using (TestServer server = new(Program.CreateWebHostBuilder(args)))
90+
{
91+
await TestRestAndGraphQLRequestsOnServerInNonHostedScenario(server);
92+
}
93+
94+
List<ITelemetry> telemetryItems = ((CustomTelemetryChannel)telemetryChannel).GetTelemetryItems();
95+
96+
// Assert that we are sending Traces/Requests/Exceptions
97+
Assert.IsTrue(telemetryItems.Any(item => item is TraceTelemetry));
98+
// Assert.IsTrue(telemetryItems.Any(item => item is RequestTelemetry));
99+
Assert.IsTrue(telemetryItems.Any(item => item is ExceptionTelemetry));
100+
101+
// Asserting on Trace telemetry items.
102+
// Checking for the Logs for the two entities Book and Publisher are correctly sent to Application Insights.
103+
Assert.IsTrue(telemetryItems.Any(item =>
104+
item is TraceTelemetry
105+
&& ((TraceTelemetry)item).Message.Equals("[Book] REST path: /api/Book")
106+
&& ((TraceTelemetry)item).SeverityLevel == SeverityLevel.Information));
107+
108+
Assert.IsTrue(telemetryItems.Any(item =>
109+
item is TraceTelemetry
110+
&& ((TraceTelemetry)item).Message.Equals("[Publisher] REST path: /api/Publisher")
111+
&& ((TraceTelemetry)item).SeverityLevel == SeverityLevel.Information));
112+
113+
// Asserting on Request telemetry items.
114+
// Assert.AreEqual(2, telemetryItems.Count(item => item is RequestTelemetry));
115+
116+
// Assert.IsTrue(telemetryItems.Any(item =>
117+
// item is RequestTelemetry
118+
// && ((RequestTelemetry)item).Name.Equals("POST /graphql")
119+
// && ((RequestTelemetry)item).ResponseCode.Equals("200")
120+
// && ((RequestTelemetry)item).Url.PathAndQuery.Equals("/graphql")));
121+
122+
// Assert.IsTrue(telemetryItems.Any(item =>
123+
// item is RequestTelemetry
124+
// && ((RequestTelemetry)item).Name.Equals("POST Rest/Insert [route]")
125+
// && ((RequestTelemetry)item).ResponseCode.Equals("403")
126+
// && ((RequestTelemetry)item).Url.PathAndQuery.Equals("/api/Publisher/id/1?name=Test")));
127+
128+
// Assert on the Exceptions telemetry items.
129+
Assert.AreEqual(1, telemetryItems.Count(item => item is ExceptionTelemetry));
130+
Assert.IsTrue(telemetryItems.Any(item =>
131+
item is ExceptionTelemetry
132+
&& ((ExceptionTelemetry)item).Message.Equals("Authorization Failure: Access Not Allowed.")));
133+
}
134+
135+
/// <summary>
136+
/// Validates that no telemetry data is sent to CustomTelemetryChannel when
137+
/// Appsights is disabled OR when no valid connectionstring is provided.
138+
/// </summary>
139+
/// <param name="isTelemetryEnabled">Whether telemetry is enabled or not.</param>
140+
/// <param name="telemetryConnectionString">Telemetry connection string.</param>
141+
[DataTestMethod]
142+
[DataRow(false, "", DisplayName = "Configuration without a connection string and with Application Insights disabled.")]
143+
[DataRow(true, "", DisplayName = "Configuration without a connection string, but with Application Insights enabled.")]
144+
[DataRow(false, TEST_APP_INSIGHTS_CONN_STRING, DisplayName = "Configuration with a connection string, but with Application Insights disabled.")]
145+
public async Task TestNoTelemetryItemsSentWhenDisabled_NonHostedScenario(bool isTelemetryEnabled, string telemetryConnectionString)
146+
{
147+
SetUpTelemetryInConfig(CONFIG_WITHOUT_TELEMETRY, isTelemetryEnabled, telemetryConnectionString);
148+
149+
string[] args = new[]
150+
{
151+
$"--ConfigFileName={CONFIG_WITHOUT_TELEMETRY}"
152+
};
153+
154+
ITelemetryChannel telemetryChannel = new CustomTelemetryChannel();
155+
Startup.CustomTelemetryChannel = telemetryChannel;
156+
157+
using (TestServer server = new(Program.CreateWebHostBuilder(args)))
158+
{
159+
await TestRestAndGraphQLRequestsOnServerInNonHostedScenario(server);
160+
}
161+
162+
List<ITelemetry> telemetryItems = ((CustomTelemetryChannel)telemetryChannel).GetTelemetryItems();
163+
164+
// Assert that we are not sending any Traces/Requests/Exceptions to Telemetry
165+
Assert.IsTrue(telemetryItems.IsNullOrEmpty());
166+
}
167+
168+
/// <summary>
169+
/// This method is just used as helper for other test methods to execute REST and GRaphQL requests
170+
/// which trigger the logging system to emit logs.
171+
/// </summary>
172+
private static async Task TestRestAndGraphQLRequestsOnServerInNonHostedScenario(TestServer server)
173+
{
174+
using (HttpClient client = server.CreateClient())
175+
{
176+
string query = @"{
177+
book_by_pk(id: 1) {
178+
id,
179+
title,
180+
publisher_id
181+
}
182+
}";
183+
184+
object payload = new { query };
185+
186+
HttpRequestMessage graphQLRequest = new(HttpMethod.Post, "/graphql")
187+
{
188+
Content = JsonContent.Create(payload)
189+
};
190+
191+
await client.SendAsync(graphQLRequest);
192+
193+
// POST request on non-accessible entity
194+
HttpRequestMessage restRequest = new(HttpMethod.Post, "/api/Publisher/id/1?name=Test");
195+
await client.SendAsync(restRequest);
196+
}
197+
}
198+
199+
/// <summary>
200+
/// The class is a custom telemetry channel to capture telemetry items and assert on them.
201+
/// </summary>
202+
private class CustomTelemetryChannel : ITelemetryChannel
203+
{
204+
private List<ITelemetry> _telemetryItems = new();
205+
206+
public CustomTelemetryChannel()
207+
{ }
208+
209+
public bool? DeveloperMode { get; set; }
210+
211+
public string EndpointAddress { get; set; }
212+
213+
public void Dispose()
214+
{ }
215+
216+
public void Flush()
217+
{
218+
}
219+
220+
public void Send(ITelemetry item)
221+
{
222+
_telemetryItems.Add(item);
223+
}
224+
225+
public List<ITelemetry> GetTelemetryItems()
226+
{
227+
return _telemetryItems;
228+
}
229+
}
230+
}

src/Service/Azure.DataApiBuilder.Service.csproj

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@
5252
<PackageReference Include="HotChocolate.AspNetCore" />
5353
<PackageReference Include="HotChocolate.AspNetCore.Authorization" />
5454
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" />
55+
<PackageReference Include="Microsoft.ApplicationInsights" />
56+
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" />
57+
<PackageReference Include="Microsoft.Extensions.Logging.ApplicationInsights" />
5558
<PackageReference Include="Microsoft.Azure.Cosmos" />
5659
<PackageReference Include="Microsoft.Data.SqlClient" />
5760
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" />

src/Service/Program.cs

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,13 @@
66
using System.CommandLine.Parsing;
77
using Azure.DataApiBuilder.Config;
88
using Azure.DataApiBuilder.Service.Exceptions;
9+
using Microsoft.ApplicationInsights;
910
using Microsoft.AspNetCore;
1011
using Microsoft.AspNetCore.Hosting;
1112
using Microsoft.Extensions.Configuration;
1213
using Microsoft.Extensions.Hosting;
1314
using Microsoft.Extensions.Logging;
15+
using Microsoft.Extensions.Logging.ApplicationInsights;
1416

1517
namespace Azure.DataApiBuilder.Service
1618
{
@@ -110,7 +112,8 @@ private static ParseResult GetParseResult(Command cmd, string[] args)
110112
/// Creates a LoggerFactory and add filter with the given LogLevel.
111113
/// </summary>
112114
/// <param name="logLevel">minimum log level.</param>
113-
public static ILoggerFactory GetLoggerFactoryForLogLevel(LogLevel logLevel)
115+
/// <param name="appTelemetryClient">Telemetry client</param>
116+
public static ILoggerFactory GetLoggerFactoryForLogLevel(LogLevel logLevel, TelemetryClient? appTelemetryClient = null)
114117
{
115118
return LoggerFactory
116119
.Create(builder =>
@@ -121,6 +124,23 @@ public static ILoggerFactory GetLoggerFactoryForLogLevel(LogLevel logLevel)
121124
builder.AddFilter(category: "Microsoft", logLevel);
122125
builder.AddFilter(category: "Azure", logLevel);
123126
builder.AddFilter(category: "Default", logLevel);
127+
128+
// For Sending all the ILogger logs to Application Insights
129+
if (Startup.AppInsightsOptions.Enabled && !string.IsNullOrWhiteSpace(Startup.AppInsightsOptions.ConnectionString))
130+
{
131+
builder.AddApplicationInsights(configureTelemetryConfiguration: (config) =>
132+
{
133+
config.ConnectionString = Startup.AppInsightsOptions.ConnectionString;
134+
if (Startup.CustomTelemetryChannel is not null)
135+
{
136+
config.TelemetryChannel = Startup.CustomTelemetryChannel;
137+
}
138+
},
139+
configureApplicationInsightsLoggerOptions: (options) => { }
140+
)
141+
.AddFilter<ApplicationInsightsLoggerProvider>(category: string.Empty, logLevel);
142+
}
143+
124144
builder.AddConsole();
125145
});
126146
}

0 commit comments

Comments
 (0)