fix: critical performance optimizations for large datasets

src/comment_provider/conversationProvider.ts

@@ -699,6 +699,12 @@ export class ConversationProvider implements Disposable {
   }
 
   dispose() {
+    // Clear any active timer to prevent memory leaks
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = undefined;
+    }
+
     while (this.disposables.length) {
       const x = this.disposables.pop();
       if (x) {

src/dbt_client/dbtCoreIntegration.ts

@@ -40,7 +40,8 @@ import { CommandProcessExecutionFactory } from "../commandProcessExecution";
 import { PythonBridge, PythonException } from "python-bridge";
 import * as path from "path";
 import { DBTProject } from "../manifest/dbtProject";
-import { existsSync, readFileSync } from "fs";
+import { readFile } from "fs/promises";
+import { existsSync } from "fs";
 import { parse } from "yaml";
 import { TelemetryService } from "../telemetry";
 import {
@@ -147,16 +148,17 @@ export class DBTCoreProjectDetection
     private dbtTerminal: DBTTerminal,
   ) {}
 
-  private getPackageInstallPathFallback(
+  private async getPackageInstallPathFallback(
     projectDirectory: Uri,
     packageInstallPath: string,
-  ): string {
+  ): Promise<string> {
     const dbtProjectFile = path.join(
       projectDirectory.fsPath,
       "dbt_project.yml",
     );
     if (existsSync(dbtProjectFile)) {
-      const dbtProjectConfig: any = parse(readFileSync(dbtProjectFile, "utf8"));
+      const fileContent = await readFile(dbtProjectFile, "utf8");
+      const dbtProjectConfig: any = parse(fileContent);
       const packagesInstallPath = dbtProjectConfig["packages-install-path"];
       if (packagesInstallPath) {
         if (path.isAbsolute(packagesInstallPath)) {
@@ -202,10 +204,12 @@ export class DBTCoreProjectDetection
         "An error occured while finding package paths: " + error,
       );
       // Fallback to reading yaml files
-      packagesInstallPaths = projectDirectories.map((projectDirectory, idx) =>
-        this.getPackageInstallPathFallback(
-          projectDirectory,
-          packagesInstallPaths[idx],
+      packagesInstallPaths = await Promise.all(
+        projectDirectories.map((projectDirectory, idx) =>
+          this.getPackageInstallPathFallback(
+            projectDirectory,
+            packagesInstallPaths[idx],
+          ),
         ),
       );
     } finally {

src/services/dbtTestService.ts

@@ -12,7 +12,7 @@ import path = require("path");
 import { DBTTerminal } from "../dbt_client/dbtTerminal";
 import { MacroMetaMap, TestMetaData } from "../domain";
 import { parse, stringify } from "yaml";
-import { readFileSync } from "fs";
+import { readFile } from "fs/promises";
 
 @provideSingleton(DbtTestService)
 export class DbtTestService {
@@ -82,7 +82,7 @@ export class DbtTestService {
   /**
    * Find the extra config for test from schema.yml, if available
    */
-  public getConfigByTest(
+  public async getConfigByTest(
     test: TestMetaData,
     modelName: string,
     columnNameFromTestMetadata?: string,
@@ -120,8 +120,9 @@ export class DbtTestService {
       "finding test from yaml",
       patchPath,
     );
+    const fileContent = await readFile(patchPath, { encoding: "utf-8" });
     const parsedDocFile = parse(
-      readFileSync(patchPath, { encoding: "utf-8" }),
+      fileContent,
       {
         strict: false,
         uniqueKeys: false,

src/webview_provider/docsEditPanel.ts

@@ -1,4 +1,5 @@
-import { existsSync, readFileSync, writeFileSync } from "fs";
+import { existsSync } from "fs";
+import { readFile, writeFile } from "fs/promises";
 import {
   CancellationToken,
   ColorThemeKind,
@@ -219,13 +220,13 @@
   ) {
     this._panel = panel;
     this.setupWebviewOptions(context);
-    this.renderWebviewView(context);
+    await this.renderWebviewView(context);
     this.updateGraphStyle();
   }
 
-  private renderWebviewView(context: WebviewViewResolveContext) {
+  private async renderWebviewView(context: WebviewViewResolveContext) {
     const webview = this._panel!.webview!;
-    webview.html = getHtml(webview, this.dbtProjectContainer.extensionUri);
+    webview.html = await getHtml(webview, this.dbtProjectContainer.extensionUri);
   }
 
   private setupWebviewOptions(context: WebviewViewResolveContext) {
@@ -425,7 +426,7 @@
     });
   };
 
-  private convertColumnNamesByCaseConfig(
+  private async convertColumnNamesByCaseConfig(
     columns: { name: string }[],
     modelName: string,
     project: DBTProject,
@@ -440,9 +441,10 @@
       return columns;
     }
 
-    const docFile: string = readFileSync(
+    const docFile: string = await readFile(
       path.join(project.projectRoot.fsPath, patchPath.split("://")[1]),
-    ).toString("utf8");
+      "utf8"
+    );
     const parsedDocFile =
       parse(docFile, {
         strict: false,
@@ -507,7 +509,7 @@
                 try {
                   const columnsInRelation =
                     await project.getColumnsOfModel(modelName);
-                  const columns = this.convertColumnNamesByCaseConfig(
+                  const columns = await this.convertColumnNamesByCaseConfig(
                     columnsInRelation.map((column) => {
                       return {
                         name: column.column,
@@ -650,11 +652,10 @@
                   }
                   // check if file exists, if not create an empty file
                   if (!existsSync(patchPath)) {
-                    writeFileSync(patchPath, "");
+                    await writeFile(patchPath, "");
                   }
 
-                  const docFile: string =
-                    readFileSync(patchPath).toString("utf8");
+                  const docFile: string = await readFile(patchPath, "utf8");
                   const parsedDocFile =
                     parse(docFile, {
                       strict: false,
@@ -754,7 +755,7 @@
                   }
                   // Force reload from manifest after manifest refresh
                   this.loadedFromManifest = false;
-                  writeFileSync(patchPath, stringify(parsedDocFile));
+                  await writeFile(patchPath, stringify(parsedDocFile));
                   this.documentation = (
                     await this.docGenService.getDocumentationForCurrentActiveFile()
                   ).documentation;
@@ -834,7 +835,7 @@
   }
 }
 
-function getHtml(webview: Webview, extensionUri: Uri) {
+async function getHtml(webview: Webview, extensionUri: Uri) {
   const indexPath = getUri(webview, extensionUri, [
     "docs_edit_panel",
     "index.html",
@@ -846,8 +847,8 @@
   ].includes(window.activeColorTheme.kind)
     ? "light"
     : "dark";
-  return readFileSync(indexPath.fsPath)
-    .toString()
+  const htmlContent = await readFile(indexPath.fsPath, "utf8");
+  return htmlContent
     .replace(/__ROOT__/g, resourceDir.toString())
     .replace(/__THEME__/g, theme)
     .replace(/__NONCE__/g, getNonce())

src/webview_provider/newDocsGenPanel.ts

@@ -1,4 +1,4 @@
-import { readFileSync } from "fs";
+import { readFile } from "fs/promises";
 import {
   CancellationToken,
   Range,
@@ -95,7 +95,7 @@ export class NewDocsGenPanel
     super.resolveWebviewView(panel, context, token);
   }
 
-  private getDbtTestCode(test: TestMetaData, modelName: string) {
+  private async getDbtTestCode(test: TestMetaData, modelName: string) {
     const { path: testPath, column_name } = test;
     this.dbtTerminal.debug(
       "getDbtTestCode",
@@ -107,9 +107,9 @@ export class NewDocsGenPanel
 
     return {
       sql: testPath?.endsWith(".sql")
-        ? readFileSync(testPath, { encoding: "utf-8" })
+        ? await readFile(testPath, { encoding: "utf-8" })
         : undefined,
-      config: this.dbtTestService.getConfigByTest(test, modelName, column_name),
+      config: await this.dbtTestService.getConfigByTest(test, modelName, column_name),
     };
   }
 
@@ -166,7 +166,7 @@ export class NewDocsGenPanel
         this.handleSyncRequestFromWebview(
           syncRequestId,
           async () => {
-            return this.getDbtTestCode(
+            return await this.getDbtTestCode(
               args.test as TestMetaData,
               args.model as string,
             );

src/webview_provider/queryResultPanel.ts

@@ -418,12 +418,15 @@ export class QueryResultPanel extends AltimateWebviewProvider {
   /** A wrapper for {@link transmitData} which converts server
    * results interface ({@link ExecuteSQLResult}) to what the webview expects */
   private async transmitDataWrapper(result: ExecuteSQLResult, query: string) {
-    const rows: JsonObj[] = [];
-    // Convert compressed array format to dict[]
+    const rows: JsonObj[] = new Array(result.table.rows.length);
+    // Convert compressed array format to dict[] - optimized version
     for (let i = 0; i < result.table.rows.length; i++) {
-      result.table.rows[i].forEach((value: any, j: any) => {
-        rows[i] = { ...rows[i], [result.table.column_names[j]]: value };
-      });
+      const row: JsonObj = {};
+      const currentRow = result.table.rows[i];
+      for (let j = 0; j < currentRow.length; j++) {
+        row[result.table.column_names[j]] = currentRow[j];
+      }
+      rows[i] = row;
     }
     await this.transmitData(
       result.table.column_names,