Optimize performance of MSGraphDb generation (#1375)

* Prevent MSGraphDb generation every run from IDE

* Disable journaling while filling data

* Wrap multiple INSERT statements within a single transaction

* Adjust parameterized insert-command

* Fix filestream resource leak

* Remove unnecessary blank lines

---------

Co-authored-by: Waldek Mastykarz <[email protected]>

Author: bartizan

DevProxy.Abstractions/Data/MSGraphDb.cs

@@ -65,7 +65,10 @@ public async Task<int> GenerateDbAsync(bool skipIfUpdatedToday, CancellationToke
             }
 
             await CreateDbAsync(cancellationToken);
+
+            SetDbJournaling(false);
             await FillDataAsync(cancellationToken);
+            SetDbJournaling(true);
 
             _logger.LogInformation("Microsoft Graph database successfully updated");
 
@@ -107,6 +110,8 @@ private async Task FillDataAsync(CancellationToken cancellationToken)
     {
         _logger.LogInformation("Filling database...");
 
+        await using var transaction = await Connection.BeginTransactionAsync(cancellationToken);
+
         var i = 0;
 
         foreach (var openApiDocument in _openApiDocuments)
@@ -120,9 +125,10 @@ private async Task FillDataAsync(CancellationToken cancellationToken)
 
             var insertEndpoint = Connection.CreateCommand();
             insertEndpoint.CommandText = "INSERT INTO endpoints (path, graphVersion, hasSelect) VALUES (@path, @graphVersion, @hasSelect)";
-            _ = insertEndpoint.Parameters.Add(new("@path", null));
-            _ = insertEndpoint.Parameters.Add(new("@graphVersion", null));
-            _ = insertEndpoint.Parameters.Add(new("@hasSelect", null));
+            var pathParam = insertEndpoint.Parameters.Add(new("@path", null));
+            var graphVersionParam = insertEndpoint.Parameters.Add(new("@graphVersion", null));
+            var hasSelectParam = insertEndpoint.Parameters.Add(new("@hasSelect", null));
+            await insertEndpoint.PrepareAsync(cancellationToken);
 
             foreach (var path in document.Paths)
             {
@@ -142,14 +148,16 @@ private async Task FillDataAsync(CancellationToken cancellationToken)
                 var hasSelect = getOperation.Parameters.Any(p => p.Name == "$select");
 
                 _logger.LogTrace("Inserting endpoint {GraphVersion}{Key} with hasSelect={HasSelect}...", graphVersion, path.Key, hasSelect);
-                insertEndpoint.Parameters["@path"].Value = path.Key;
-                insertEndpoint.Parameters["@graphVersion"].Value = graphVersion;
-                insertEndpoint.Parameters["@hasSelect"].Value = hasSelect;
+                pathParam.Value = path.Key;
+                graphVersionParam.Value = graphVersion;
+                hasSelectParam.Value = hasSelect;
                 _ = await insertEndpoint.ExecuteNonQueryAsync(cancellationToken);
                 i++;
             }
         }
 
+        await transaction.CommitAsync(cancellationToken);
+
         _logger.LogInformation("Inserted {EndpointCount} endpoints in the database", i);
     }
 
@@ -202,7 +210,8 @@ private async Task LoadOpenAPIFilesAsync(string folder, CancellationToken cancel
 
             try
             {
-                var openApiDocument = await new OpenApiStreamReader().ReadAsync(file.OpenRead(), cancellationToken);
+                await using var fileStream = file.OpenRead();
+                var openApiDocument = await new OpenApiStreamReader().ReadAsync(fileStream, cancellationToken);
                 _openApiDocuments[version] = openApiDocument.OpenApiDocument;
 
                 _logger.LogDebug("Added OpenAPI file {FilePath} for {Version}", filePath, version);
@@ -214,6 +223,21 @@ private async Task LoadOpenAPIFilesAsync(string folder, CancellationToken cancel
         }
     }
 
+    private void SetDbJournaling(bool enabled)
+    {
+        using var command = Connection.CreateCommand();
+        if (enabled)
+        {
+            command.CommandText = "PRAGMA journal_mode = DELETE;";
+            _ = command.ExecuteNonQuery();
+        }
+        else
+        {
+            command.CommandText = "PRAGMA journal_mode = OFF;";
+            _ = command.ExecuteNonQuery();
+        }
+    }
+
     public void Dispose()
     {
         _connection?.Dispose();

DevProxy/DevProxy.csproj

@@ -57,7 +57,7 @@
 
   <ItemGroup>
     <None Update="msgraph-openapi-v1.db">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
     <None Update="devproxyrc.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>