Optimize MSGraph OpenApi specs caching based on e-tag (#1379)

* Extract method to get an api URL

* Arrange functions to get yaml and e-tag open api filenames

* Correct some logging messages

* Move SetDbJournaling()

* Add second check if db should be updated

* Extract IsModifiedToday function

* Extract DownloadOpenAPIFileAsync()

* Add conditional downloading by an e-tag

* Add debug logging for last-write-time attr update

* Prevent db generation if a spec update fails

* Update DevProxy.Abstractions/Data/MSGraphDb.cs

Co-authored-by: Copilot <[email protected]>

* Update DevProxy.Abstractions/Data/MSGraphDb.cs

Co-authored-by: Copilot <[email protected]>

* Fix discarded variable declaration

* Minor fixes

---------

Co-authored-by: Waldek Mastykarz <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: 3 people

DevProxy.Abstractions/Data/MSGraphDb.cs

@@ -7,6 +7,8 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.OpenApi.Models;
 using Microsoft.OpenApi.Readers;
+using System.Net;
+using System.Net.Http.Headers;
 
 namespace DevProxy.Abstractions.Data;
 
@@ -49,14 +51,29 @@ public async Task<int> GenerateDbAsync(bool skipIfUpdatedToday, CancellationToke
         try
         {
             var dbFileInfo = new FileInfo(MSGraphDbFilePath);
-            var modifiedToday = dbFileInfo.Exists && dbFileInfo.LastWriteTime.Date == DateTime.Now.Date;
+            var modifiedToday = IsModifiedToday(dbFileInfo);
             if (modifiedToday && skipIfUpdatedToday)
             {
-                _logger.LogInformation("Microsoft Graph database already updated today");
+                _logger.LogInformation("Microsoft Graph database has already been updated today");
+                return 1;
+            }
+
+            var (isApiModified, hasErrors) = await UpdateOpenAPIGraphFilesIfNecessaryAsync(appFolder, cancellationToken);
+
+            if (hasErrors)
+            {
+                _logger.LogWarning("Unable to update Microsoft Graph database");
+                return 1;
+            }
+
+            if (!isApiModified)
+            {
+                UpdateLastWriteTime(dbFileInfo);
+                _logger.LogDebug("Updated the last-write-time attribute of Microsoft Graph database {File}", dbFileInfo);
+                _logger.LogInformation("Microsoft Graph database is already up-to-date");
                 return 1;
             }
 
-            await UpdateOpenAPIGraphFilesIfNecessaryAsync(appFolder, cancellationToken);
             await LoadOpenAPIFilesAsync(appFolder, cancellationToken);
             if (_openApiDocuments.Count < 1)
             {
@@ -65,24 +82,21 @@ public async Task<int> GenerateDbAsync(bool skipIfUpdatedToday, CancellationToke
             }
 
             await CreateDbAsync(cancellationToken);
-
-            SetDbJournaling(false);
             await FillDataAsync(cancellationToken);
-            SetDbJournaling(true);
-
-            _logger.LogInformation("Microsoft Graph database successfully updated");
 
+            _logger.LogInformation("Microsoft Graph database is successfully updated");
             return 0;
         }
         catch (Exception ex)
         {
             _logger.LogError(ex, "Error generating Microsoft Graph database");
             return 1;
         }
-
     }
 
-    private static string GetGraphOpenApiYamlFileName(string version) => $"graph-{version.Replace(".", "_", StringComparison.OrdinalIgnoreCase)}-openapi.yaml";
+    private static bool IsModifiedToday(FileInfo fileInfo) => fileInfo.Exists && fileInfo.LastWriteTime.Date == DateTime.Now.Date;
+
+    private static void UpdateLastWriteTime(FileInfo fileInfo) => fileInfo.LastWriteTime = DateTime.Now;
 
     private async Task CreateDbAsync(CancellationToken cancellationToken)
     {
@@ -110,6 +124,8 @@ private async Task FillDataAsync(CancellationToken cancellationToken)
     {
         _logger.LogInformation("Filling database...");
 
+        SetDbJournaling(false);
+
         await using var transaction = await Connection.BeginTransactionAsync(cancellationToken);
 
         var i = 0;
@@ -158,38 +174,86 @@ private async Task FillDataAsync(CancellationToken cancellationToken)
 
         await transaction.CommitAsync(cancellationToken);
 
+        SetDbJournaling(true);
+
         _logger.LogInformation("Inserted {EndpointCount} endpoints in the database", i);
     }
 
-    private async Task UpdateOpenAPIGraphFilesIfNecessaryAsync(string folder, CancellationToken cancellationToken)
+    private async Task<(bool isApiUpdated, bool hasErrors)> UpdateOpenAPIGraphFilesIfNecessaryAsync(string folder, CancellationToken cancellationToken)
     {
         _logger.LogInformation("Checking for updated OpenAPI files...");
 
+        var isApiUpdated = false;
+        var hasErrors = false;
+
         foreach (var version in graphVersions)
         {
             try
             {
-                var file = new FileInfo(Path.Combine(folder, GetGraphOpenApiYamlFileName(version)));
-                _logger.LogDebug("Checking for updated OpenAPI file {File}...", file);
-                if (file.Exists && file.LastWriteTime.Date == DateTime.Now.Date)
+                var yamlFile = new FileInfo(Path.Combine(folder, GetGraphOpenApiYamlFileName(version)));
+                _logger.LogDebug("Checking for updated OpenAPI file {File}...", yamlFile);
+                if (IsModifiedToday(yamlFile))
                 {
-                    _logger.LogInformation("File {File} already updated today", file);
+                    _logger.LogInformation("File {File} has already been updated today", yamlFile);
                     continue;
                 }
 
-                var url = $"https://raw.githubusercontent.com/microsoftgraph/msgraph-metadata/master/openapi/{version}/openapi.yaml";
+                var url = GetOpenApiSpecUrl(version);
                 _logger.LogInformation("Downloading OpenAPI file from {Url}...", url);
 
-                var response = await _httpClient.GetStringAsync(url, cancellationToken);
-                await File.WriteAllTextAsync(file.FullName, response, cancellationToken);
-
-                _logger.LogDebug("Downloaded OpenAPI file from {Url} to {File}", url, file);
+                var etagFile = new FileInfo(Path.Combine(folder, GetGraphOpenApiEtagFileName(version)));
+                isApiUpdated |= await DownloadOpenAPIFileAsync(url, yamlFile, etagFile, cancellationToken);
             }
             catch (Exception ex)
             {
+                hasErrors = true;
                 _logger.LogError(ex, "Error updating OpenAPI files");
             }
         }
+        return (isApiUpdated, hasErrors);
+    }
+
+    private async Task<bool> DownloadOpenAPIFileAsync(string url, FileInfo yamlFile, FileInfo etagFile, CancellationToken cancellationToken)
+    {
+        var tag = string.Empty;
+        if (etagFile.Exists)
+        {
+            tag = await File.ReadAllTextAsync(etagFile.FullName, cancellationToken);
+        }
+
+        using var requestMessage = new HttpRequestMessage(HttpMethod.Get, url);
+        if (!string.IsNullOrWhiteSpace(tag))
+        {
+            requestMessage.Headers.IfNoneMatch.Add(new EntityTagHeaderValue(tag));
+        }
+
+        var response = await _httpClient.SendAsync(requestMessage, cancellationToken);
+
+        if (response.StatusCode == HttpStatusCode.NotModified)
+        {
+            UpdateLastWriteTime(yamlFile);
+            _logger.LogDebug("File {File} already up-to-date. Updated the last-write-time attribute", yamlFile);
+            return false;
+        }
+
+        // Save the new OpenAPI spec.
+        _ = response.EnsureSuccessStatusCode();
+        await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
+        await using var fileStream = new FileStream(yamlFile.FullName,
+            new FileStreamOptions { Mode = FileMode.Create, Access = FileAccess.Write, Share = FileShare.None });
+        await contentStream.CopyToAsync(fileStream, cancellationToken);
+
+        if (response.Headers.ETag is not null)
+        {
+            await File.WriteAllTextAsync(etagFile.FullName, response.Headers.ETag.Tag, cancellationToken);
+        }
+        else
+        {
+            etagFile.Delete();
+        }
+
+        _logger.LogDebug("Downloaded OpenAPI file from {Url} to {File}", url, yamlFile);
+        return true;
     }
 
     private async Task LoadOpenAPIFilesAsync(string folder, CancellationToken cancellationToken)
@@ -238,6 +302,14 @@ private void SetDbJournaling(bool enabled)
         }
     }
 
+    private static string GetOpenApiSpecUrl(string version) => $"https://raw.githubusercontent.com/microsoftgraph/msgraph-metadata/master/openapi/{version}/openapi.yaml";
+
+    private static string GetBaseGraphOpenApiFileName(string version) => $"graph-{version.Replace(".", "_", StringComparison.OrdinalIgnoreCase)}-openapi";
+
+    private static string GetGraphOpenApiYamlFileName(string version) => $"{GetBaseGraphOpenApiFileName(version)}.yaml";
+
+    private static string GetGraphOpenApiEtagFileName(string version) => $"{GetBaseGraphOpenApiFileName(version)}.etag.txt";
+
     public void Dispose()
     {
         _connection?.Dispose();