diff --git a/DevProxy.Abstractions/Utils/MSGraphDbUtils.cs b/DevProxy.Abstractions/Data/MSGraphDb.cs similarity index 54% rename from DevProxy.Abstractions/Utils/MSGraphDbUtils.cs rename to DevProxy.Abstractions/Data/MSGraphDb.cs index 236e6ba7..9ce5c021 100644 --- a/DevProxy.Abstractions/Utils/MSGraphDbUtils.cs +++ b/DevProxy.Abstractions/Data/MSGraphDb.cs @@ -1,214 +1,221 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Data.Sqlite; -using Microsoft.Extensions.Logging; -using Microsoft.OpenApi.Models; -using Microsoft.OpenApi.Readers; - -namespace DevProxy.Abstractions.Utils; - -public static class MSGraphDbUtils -{ - private static readonly Dictionary _openApiDocuments = []; - private static readonly string[] graphVersions = ["v1.0", "beta"]; - - private static SqliteConnection? _msGraphDbConnection; - - // v1 refers to v1 of the db schema, not the graph version - public static string MSGraphDbFilePath => Path.Combine(ProxyUtils.AppFolder!, "msgraph-openapi-v1.db"); - - public static SqliteConnection MSGraphDbConnection - { - get - { - if (_msGraphDbConnection is null) - { - _msGraphDbConnection = new($"Data Source={MSGraphDbFilePath}"); - _msGraphDbConnection.Open(); - } - - return _msGraphDbConnection; - } - } - - public static async Task GenerateMSGraphDbAsync(ILogger logger, bool skipIfUpdatedToday, CancellationToken cancellationToken) - { - var appFolder = ProxyUtils.AppFolder; - if (string.IsNullOrEmpty(appFolder)) - { - logger.LogError("App folder {AppFolder} not found", appFolder); - return 1; - } - - try - { - var dbFileInfo = new FileInfo(MSGraphDbFilePath); - var modifiedToday = dbFileInfo.Exists && dbFileInfo.LastWriteTime.Date == DateTime.Now.Date; - if (modifiedToday && skipIfUpdatedToday) - { - logger.LogInformation("Microsoft Graph database already updated today"); - return 1; - } - - await UpdateOpenAPIGraphFilesIfNecessaryAsync(appFolder, logger, cancellationToken); - await LoadOpenAPIFilesAsync(appFolder, logger, cancellationToken); - if (_openApiDocuments.Count < 1) - { - logger.LogDebug("No OpenAPI files found or couldn't load them"); - return 1; - } - - var dbConnection = MSGraphDbConnection; - await CreateDbAsync(dbConnection, logger, cancellationToken); - await FillDataAsync(dbConnection, logger, cancellationToken); - - logger.LogInformation("Microsoft Graph database successfully updated"); - - return 0; - } - catch (Exception ex) - { - logger.LogError(ex, "Error generating Microsoft Graph database"); - return 1; - } - - } - - private static string GetGraphOpenApiYamlFileName(string version) => $"graph-{version.Replace(".", "_", StringComparison.OrdinalIgnoreCase)}-openapi.yaml"; - - private static async Task CreateDbAsync(SqliteConnection dbConnection, ILogger logger, CancellationToken cancellationToken) - { - logger.LogInformation("Creating database..."); - - logger.LogDebug("Dropping endpoints table..."); - var dropTable = dbConnection.CreateCommand(); - dropTable.CommandText = "DROP TABLE IF EXISTS endpoints"; - _ = await dropTable.ExecuteNonQueryAsync(cancellationToken); - - logger.LogDebug("Creating endpoints table..."); - var createTable = dbConnection.CreateCommand(); - // when you change the schema, increase the db version number in ProxyUtils - createTable.CommandText = "CREATE TABLE IF NOT EXISTS endpoints (path TEXT, graphVersion TEXT, hasSelect BOOLEAN)"; - _ = await createTable.ExecuteNonQueryAsync(cancellationToken); - - logger.LogDebug("Creating index on endpoints and version..."); - // Add an index on the path and graphVersion columns - var createIndex = dbConnection.CreateCommand(); - createIndex.CommandText = "CREATE INDEX IF NOT EXISTS idx_endpoints_path_version ON endpoints (path, graphVersion)"; - _ = await createIndex.ExecuteNonQueryAsync(cancellationToken); - } - - private static async Task FillDataAsync(SqliteConnection dbConnection, ILogger logger, CancellationToken cancellationToken) - { - logger.LogInformation("Filling database..."); - - var i = 0; - - foreach (var openApiDocument in _openApiDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var graphVersion = openApiDocument.Key; - var document = openApiDocument.Value; - - logger.LogDebug("Filling database for {GraphVersion}...", graphVersion); - - var insertEndpoint = dbConnection.CreateCommand(); - insertEndpoint.CommandText = "INSERT INTO endpoints (path, graphVersion, hasSelect) VALUES (@path, @graphVersion, @hasSelect)"; - _ = insertEndpoint.Parameters.Add(new("@path", null)); - _ = insertEndpoint.Parameters.Add(new("@graphVersion", null)); - _ = insertEndpoint.Parameters.Add(new("@hasSelect", null)); - - foreach (var path in document.Paths) - { - cancellationToken.ThrowIfCancellationRequested(); - - logger.LogTrace("Endpoint {GraphVersion}{Key}...", graphVersion, path.Key); - - // Get the GET operation for this path - var getOperation = path.Value.Operations.FirstOrDefault(o => o.Key == OperationType.Get).Value; - if (getOperation == null) - { - logger.LogTrace("No GET operation found for {GraphVersion}{Key}", graphVersion, path.Key); - continue; - } - - // Check if the GET operation has a $select parameter - var hasSelect = getOperation.Parameters.Any(p => p.Name == "$select"); - - logger.LogTrace("Inserting endpoint {GraphVersion}{Key} with hasSelect={HasSelect}...", graphVersion, path.Key, hasSelect); - insertEndpoint.Parameters["@path"].Value = path.Key; - insertEndpoint.Parameters["@graphVersion"].Value = graphVersion; - insertEndpoint.Parameters["@hasSelect"].Value = hasSelect; - _ = await insertEndpoint.ExecuteNonQueryAsync(cancellationToken); - i++; - } - } - - logger.LogInformation("Inserted {EndpointCount} endpoints in the database", i); - } - - private static async Task UpdateOpenAPIGraphFilesIfNecessaryAsync(string folder, ILogger logger, CancellationToken cancellationToken) - { - logger.LogInformation("Checking for updated OpenAPI files..."); - - foreach (var version in graphVersions) - { - try - { - var file = new FileInfo(Path.Combine(folder, GetGraphOpenApiYamlFileName(version))); - logger.LogDebug("Checking for updated OpenAPI file {File}...", file); - if (file.Exists && file.LastWriteTime.Date == DateTime.Now.Date) - { - logger.LogInformation("File {File} already updated today", file); - continue; - } - - var url = $"https://raw.githubusercontent.com/microsoftgraph/msgraph-metadata/master/openapi/{version}/openapi.yaml"; - logger.LogInformation("Downloading OpenAPI file from {Url}...", url); - - using var client = new HttpClient(); - var response = await client.GetStringAsync(url, cancellationToken); - await File.WriteAllTextAsync(file.FullName, response, cancellationToken); - - logger.LogDebug("Downloaded OpenAPI file from {Url} to {File}", url, file); - } - catch (Exception ex) - { - logger.LogError(ex, "Error updating OpenAPI files"); - } - } - } - - private static async Task LoadOpenAPIFilesAsync(string folder, ILogger logger, CancellationToken cancellationToken) - { - logger.LogInformation("Loading OpenAPI files..."); - - foreach (var version in graphVersions) - { - var filePath = Path.Combine(folder, GetGraphOpenApiYamlFileName(version)); - var file = new FileInfo(filePath); - logger.LogDebug("Loading OpenAPI file for {FilePath}...", filePath); - - if (!file.Exists) - { - logger.LogDebug("File {FilePath} does not exist", filePath); - continue; - } - - try - { - var openApiDocument = await new OpenApiStreamReader().ReadAsync(file.OpenRead(), cancellationToken); - _openApiDocuments[version] = openApiDocument.OpenApiDocument; - - logger.LogDebug("Added OpenAPI file {FilePath} for {Version}", filePath, version); - } - catch (Exception ex) - { - logger.LogError(ex, "Error loading OpenAPI file {FilePath}", filePath); - } - } - } +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Abstractions.Utils; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; +using Microsoft.OpenApi.Models; +using Microsoft.OpenApi.Readers; + +namespace DevProxy.Abstractions.Data; + +public sealed class MSGraphDb(HttpClient httpClient, ILogger logger) : IDisposable +{ + private static readonly string[] graphVersions = ["v1.0", "beta"]; + private readonly Dictionary _openApiDocuments = []; +#pragma warning disable CA2213 // Disposable fields should be disposed + private readonly HttpClient _httpClient = httpClient; +#pragma warning restore CA2213 // Disposable fields should be disposed + private readonly ILogger _logger = logger; + private SqliteConnection? _connection; + + // v1 refers to v1 of the db schema, not the graph version + public static string MSGraphDbFilePath => Path.Combine(ProxyUtils.AppFolder!, "msgraph-openapi-v1.db"); + + public SqliteConnection Connection + { + get + { + if (_connection is null) + { + _connection = new($"Data Source={MSGraphDbFilePath}"); + _connection.Open(); + } + + return _connection; + } + } + + public async Task GenerateDbAsync(bool skipIfUpdatedToday, CancellationToken cancellationToken) + { + var appFolder = ProxyUtils.AppFolder; + if (string.IsNullOrEmpty(appFolder)) + { + _logger.LogError("App folder {AppFolder} not found", appFolder); + return 1; + } + + try + { + var dbFileInfo = new FileInfo(MSGraphDbFilePath); + var modifiedToday = dbFileInfo.Exists && dbFileInfo.LastWriteTime.Date == DateTime.Now.Date; + if (modifiedToday && skipIfUpdatedToday) + { + _logger.LogInformation("Microsoft Graph database already updated today"); + return 1; + } + + await UpdateOpenAPIGraphFilesIfNecessaryAsync(appFolder, cancellationToken); + await LoadOpenAPIFilesAsync(appFolder, cancellationToken); + if (_openApiDocuments.Count < 1) + { + _logger.LogDebug("No OpenAPI files found or couldn't load them"); + return 1; + } + + await CreateDbAsync(cancellationToken); + await FillDataAsync(cancellationToken); + + _logger.LogInformation("Microsoft Graph database successfully updated"); + + return 0; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error generating Microsoft Graph database"); + return 1; + } + + } + + private static string GetGraphOpenApiYamlFileName(string version) => $"graph-{version.Replace(".", "_", StringComparison.OrdinalIgnoreCase)}-openapi.yaml"; + + private async Task CreateDbAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Creating database..."); + + _logger.LogDebug("Dropping endpoints table..."); + var dropTable = Connection.CreateCommand(); + dropTable.CommandText = "DROP TABLE IF EXISTS endpoints"; + _ = await dropTable.ExecuteNonQueryAsync(cancellationToken); + + _logger.LogDebug("Creating endpoints table..."); + var createTable = Connection.CreateCommand(); + // when you change the schema, increase the db version number in ProxyUtils + createTable.CommandText = "CREATE TABLE IF NOT EXISTS endpoints (path TEXT, graphVersion TEXT, hasSelect BOOLEAN)"; + _ = await createTable.ExecuteNonQueryAsync(cancellationToken); + + _logger.LogDebug("Creating index on endpoints and version..."); + // Add an index on the path and graphVersion columns + var createIndex = Connection.CreateCommand(); + createIndex.CommandText = "CREATE INDEX IF NOT EXISTS idx_endpoints_path_version ON endpoints (path, graphVersion)"; + _ = await createIndex.ExecuteNonQueryAsync(cancellationToken); + } + + private async Task FillDataAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Filling database..."); + + var i = 0; + + foreach (var openApiDocument in _openApiDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var graphVersion = openApiDocument.Key; + var document = openApiDocument.Value; + + _logger.LogDebug("Filling database for {GraphVersion}...", graphVersion); + + var insertEndpoint = Connection.CreateCommand(); + insertEndpoint.CommandText = "INSERT INTO endpoints (path, graphVersion, hasSelect) VALUES (@path, @graphVersion, @hasSelect)"; + _ = insertEndpoint.Parameters.Add(new("@path", null)); + _ = insertEndpoint.Parameters.Add(new("@graphVersion", null)); + _ = insertEndpoint.Parameters.Add(new("@hasSelect", null)); + + foreach (var path in document.Paths) + { + cancellationToken.ThrowIfCancellationRequested(); + + _logger.LogTrace("Endpoint {GraphVersion}{Key}...", graphVersion, path.Key); + + // Get the GET operation for this path + var getOperation = path.Value.Operations.FirstOrDefault(o => o.Key == OperationType.Get).Value; + if (getOperation == null) + { + _logger.LogTrace("No GET operation found for {GraphVersion}{Key}", graphVersion, path.Key); + continue; + } + + // Check if the GET operation has a $select parameter + var hasSelect = getOperation.Parameters.Any(p => p.Name == "$select"); + + _logger.LogTrace("Inserting endpoint {GraphVersion}{Key} with hasSelect={HasSelect}...", graphVersion, path.Key, hasSelect); + insertEndpoint.Parameters["@path"].Value = path.Key; + insertEndpoint.Parameters["@graphVersion"].Value = graphVersion; + insertEndpoint.Parameters["@hasSelect"].Value = hasSelect; + _ = await insertEndpoint.ExecuteNonQueryAsync(cancellationToken); + i++; + } + } + + _logger.LogInformation("Inserted {EndpointCount} endpoints in the database", i); + } + + private async Task UpdateOpenAPIGraphFilesIfNecessaryAsync(string folder, CancellationToken cancellationToken) + { + _logger.LogInformation("Checking for updated OpenAPI files..."); + + foreach (var version in graphVersions) + { + try + { + var file = new FileInfo(Path.Combine(folder, GetGraphOpenApiYamlFileName(version))); + _logger.LogDebug("Checking for updated OpenAPI file {File}...", file); + if (file.Exists && file.LastWriteTime.Date == DateTime.Now.Date) + { + _logger.LogInformation("File {File} already updated today", file); + continue; + } + + var url = $"https://raw.githubusercontent.com/microsoftgraph/msgraph-metadata/master/openapi/{version}/openapi.yaml"; + _logger.LogInformation("Downloading OpenAPI file from {Url}...", url); + + var response = await _httpClient.GetStringAsync(url, cancellationToken); + await File.WriteAllTextAsync(file.FullName, response, cancellationToken); + + _logger.LogDebug("Downloaded OpenAPI file from {Url} to {File}", url, file); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error updating OpenAPI files"); + } + } + } + + private async Task LoadOpenAPIFilesAsync(string folder, CancellationToken cancellationToken) + { + _logger.LogInformation("Loading OpenAPI files..."); + + foreach (var version in graphVersions) + { + var filePath = Path.Combine(folder, GetGraphOpenApiYamlFileName(version)); + var file = new FileInfo(filePath); + _logger.LogDebug("Loading OpenAPI file for {FilePath}...", filePath); + + if (!file.Exists) + { + _logger.LogDebug("File {FilePath} does not exist", filePath); + continue; + } + + try + { + var openApiDocument = await new OpenApiStreamReader().ReadAsync(file.OpenRead(), cancellationToken); + _openApiDocuments[version] = openApiDocument.OpenApiDocument; + + _logger.LogDebug("Added OpenAPI file {FilePath} for {Version}", filePath, version); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error loading OpenAPI file {FilePath}", filePath); + } + } + } + + public void Dispose() + { + _connection?.Dispose(); + } } \ No newline at end of file diff --git a/DevProxy.Plugins/Guidance/GraphSelectGuidancePlugin.cs b/DevProxy.Plugins/Guidance/GraphSelectGuidancePlugin.cs index a41d524c..9f41574e 100644 --- a/DevProxy.Plugins/Guidance/GraphSelectGuidancePlugin.cs +++ b/DevProxy.Plugins/Guidance/GraphSelectGuidancePlugin.cs @@ -2,19 +2,23 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using DevProxy.Abstractions.Data; using DevProxy.Abstractions.Proxy; using DevProxy.Abstractions.Plugins; using DevProxy.Abstractions.Utils; using Microsoft.Extensions.Logging; -using Titanium.Web.Proxy.EventArguments; using System.Globalization; +using Titanium.Web.Proxy.EventArguments; namespace DevProxy.Plugins.Guidance; public sealed class GraphSelectGuidancePlugin( ILogger logger, - ISet urlsToWatch) : BasePlugin(logger, urlsToWatch) + ISet urlsToWatch, + MSGraphDb msGraphDb) : BasePlugin(logger, urlsToWatch) { + private readonly MSGraphDb _msGraphDb = msGraphDb; + public override string Name => nameof(GraphSelectGuidancePlugin); public override async Task InitializeAsync(InitArgs e, CancellationToken cancellationToken) @@ -22,7 +26,7 @@ public override async Task InitializeAsync(InitArgs e, CancellationToken cancell await base.InitializeAsync(e, cancellationToken); // let's not await so that it doesn't block the proxy startup - _ = MSGraphDbUtils.GenerateMSGraphDbAsync(Logger, true, cancellationToken); + _ = _msGraphDb.GenerateDbAsync(true, cancellationToken); } public override Task AfterResponseAsync(ProxyResponseArgs e, CancellationToken cancellationToken) @@ -82,7 +86,7 @@ private bool EndpointSupportsSelect(string graphVersion, string relativeUrl) try { - var dbConnection = MSGraphDbUtils.MSGraphDbConnection; + var dbConnection = _msGraphDb.Connection; // lookup information from the database var selectEndpoint = dbConnection.CreateCommand(); selectEndpoint.CommandText = "SELECT hasSelect FROM endpoints WHERE path = @path AND graphVersion = @graphVersion"; diff --git a/DevProxy.Plugins/Reporting/GraphMinimalPermissionsGuidancePlugin.cs b/DevProxy.Plugins/Reporting/GraphMinimalPermissionsGuidancePlugin.cs index ca88be18..11ce07b9 100644 --- a/DevProxy.Plugins/Reporting/GraphMinimalPermissionsGuidancePlugin.cs +++ b/DevProxy.Plugins/Reporting/GraphMinimalPermissionsGuidancePlugin.cs @@ -50,6 +50,7 @@ public sealed class GraphMinimalPermissionsGuidancePlugin( pluginConfigurationSection) { private GraphUtils? _graphUtils; + private readonly HttpClient _httpClient = httpClient; public override string Name => nameof(GraphMinimalPermissionsGuidancePlugin); @@ -240,11 +241,10 @@ private async Task EvaluateMinimalScopesAsync( try { var url = $"https://devxapi-func-prod-eastus.azurewebsites.net/permissions?scopeType={GraphUtils.GetScopeTypeString(scopeType)}"; - using var client = new HttpClient(); var stringPayload = JsonSerializer.Serialize(payload, ProxyUtils.JsonSerializerOptions); Logger.LogDebug("Calling {Url} with payload{NewLine}{Payload}", url, Environment.NewLine, stringPayload); - var response = await client.PostAsJsonAsync(url, payload, cancellationToken); + var response = await _httpClient.PostAsJsonAsync(url, payload, cancellationToken); var content = await response.Content.ReadAsStringAsync(cancellationToken); Logger.LogDebug("Response:{NewLine}{Content}", Environment.NewLine, content); diff --git a/DevProxy.Plugins/Reporting/GraphMinimalPermissionsPlugin.cs b/DevProxy.Plugins/Reporting/GraphMinimalPermissionsPlugin.cs index 1f16aeb7..24b3aac1 100644 --- a/DevProxy.Plugins/Reporting/GraphMinimalPermissionsPlugin.cs +++ b/DevProxy.Plugins/Reporting/GraphMinimalPermissionsPlugin.cs @@ -35,6 +35,7 @@ public sealed class GraphMinimalPermissionsPlugin( pluginConfigurationSection) { private GraphUtils? _graphUtils; + private readonly HttpClient _httpClient = httpClient; public override string Name => nameof(GraphMinimalPermissionsPlugin); @@ -137,11 +138,10 @@ public override async Task AfterRecordingStopAsync(RecordingArgs e, Cancellation try { var url = $"https://devxapi-func-prod-eastus.azurewebsites.net/permissions?scopeType={GraphUtils.GetScopeTypeString(Configuration.Type)}"; - using var client = new HttpClient(); var stringPayload = JsonSerializer.Serialize(payload, ProxyUtils.JsonSerializerOptions); Logger.LogDebug("Calling {Url} with payload\r\n{StringPayload}", url, stringPayload); - var response = await client.PostAsJsonAsync(url, payload, cancellationToken); + var response = await _httpClient.PostAsJsonAsync(url, payload, cancellationToken); var content = await response.Content.ReadAsStringAsync(cancellationToken); Logger.LogDebug("Response:\r\n{Content}", content); diff --git a/DevProxy/Announcement.cs b/DevProxy/Announcement.cs index 9d42fc88..c200b9fd 100644 --- a/DevProxy/Announcement.cs +++ b/DevProxy/Announcement.cs @@ -22,7 +22,7 @@ public static async Task ShowAsync() } } - public static async Task GetAsync() + private static async Task GetAsync() { try { diff --git a/DevProxy/Commands/MsGraphDbCommand.cs b/DevProxy/Commands/MsGraphDbCommand.cs index ee995f44..bad97a54 100644 --- a/DevProxy/Commands/MsGraphDbCommand.cs +++ b/DevProxy/Commands/MsGraphDbCommand.cs @@ -2,19 +2,19 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using DevProxy.Abstractions.Utils; +using DevProxy.Abstractions.Data; using System.CommandLine; namespace DevProxy.Commands; sealed class MsGraphDbCommand : Command { - private readonly ILogger _logger; + private readonly MSGraphDb _msGraphDb; - public MsGraphDbCommand(ILogger logger) : + public MsGraphDbCommand(MSGraphDb msGraphDb) : base("msgraphdb", "Generate a local SQLite database with Microsoft Graph API metadata") { - _logger = logger; + _msGraphDb = msGraphDb; ConfigureCommand(); } @@ -25,6 +25,6 @@ private void ConfigureCommand() private async Task GenerateMsGraphDbAsync(ParseResult parseResult, CancellationToken cancellationToken) { - _ = await MSGraphDbUtils.GenerateMSGraphDbAsync(_logger, false, cancellationToken); + _ = await _msGraphDb.GenerateDbAsync(false, cancellationToken); } } \ No newline at end of file diff --git a/DevProxy/Extensions/IServiceCollectionExtensions.cs b/DevProxy/Extensions/IServiceCollectionExtensions.cs index 13ee3906..98bed44a 100644 --- a/DevProxy/Extensions/IServiceCollectionExtensions.cs +++ b/DevProxy/Extensions/IServiceCollectionExtensions.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using DevProxy; +using DevProxy.Abstractions.Data; using DevProxy.Abstractions.LanguageModel; using DevProxy.Abstractions.Proxy; using DevProxy.Commands; @@ -45,6 +46,7 @@ static IServiceCollection AddApplicationServices( .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddHttpClient(); _ = services.AddPlugins(configuration, options); diff --git a/dev-proxy-plugins/Mocks/MockResponsePlugin.cs b/dev-proxy-plugins/Mocks/MockResponsePlugin.cs new file mode 100644 index 00000000..e69de29b