wip: indexing

Author: KristofersOzolinsMagebit

package.json

@@ -67,6 +67,11 @@
           "type": "boolean",
           "default": true,
           "description": "Enable hover decorations for Magento 2 XML files."
+        },
+        "magento-toolbox.indexingBatchSize": {
+          "type": "number",
+          "default": 25,
+          "description": "Number of files to index in a single batch."
         }
       }
     },

src/common/Config.ts

@@ -3,8 +3,8 @@ import { workspace } from 'vscode';
 class Config {
   public readonly SECTION = 'magento-toolbox';
 
-  public get<T = string>(key: string): T | undefined {
-    return workspace.getConfiguration(this.SECTION).get<T>(key);
+  public get<T = string>(key: string, defaultValue: T): T {
+    return workspace.getConfiguration(this.SECTION).get<T>(key, defaultValue);
   }
 }
 

src/common/php/FileHeader.ts

@@ -2,7 +2,7 @@ import Config from 'common/Config';
 
 export default class FileHeader {
   public static getHeader(module: string): string | undefined {
-    const header = Config.get<string>('phpFileHeaderComment');
+    const header = Config.get<string>('phpFileHeaderComment', '');
 
     if (!header) {
       return undefined;

src/common/xml/FileHeader.ts

@@ -2,7 +2,7 @@ import Config from 'common/Config';
 
 export default class FileHeader {
   public static getHeader(module: string): string | undefined {
-    const header = Config.get<string>('xmlFileHeaderComment');
+    const header = Config.get<string>('xmlFileHeaderComment', '');
 
     if (!header) {
       return undefined;

src/common/xml/XmlSuggestionProvider.ts

@@ -134,7 +134,7 @@ export abstract class XmlSuggestionProvider<T> {
 
   public canProvideSuggestions(document: TextDocument): boolean {
     if (this.getConfigKey()) {
-      const provideXmlSuggestions = Config.get<boolean>(this.getConfigKey()!);
+      const provideXmlSuggestions = Config.get<boolean>(this.getConfigKey()!, true);
 
       if (!provideXmlSuggestions) {
         return false;

src/definition/XmlClasslikeDefinitionProvider.ts

@@ -20,7 +20,7 @@ export class XmlClasslikeDefinitionProvider implements DefinitionProvider {
     position: Position,
     token: CancellationToken
   ) {
-    const provideXmlDefinitions = Config.get<boolean>('provideXmlDefinitions');
+    const provideXmlDefinitions = Config.get<boolean>('provideXmlDefinitions', true);
 
     if (!provideXmlDefinitions) {
       return null;

src/hover/XmlClasslikeHoverProvider.ts

@@ -8,7 +8,7 @@ import { Hover, HoverProvider, Position, Range, TextDocument } from 'vscode';
 
 export default class XmlClasslikeHoverProvider implements HoverProvider {
   public async provideHover(document: TextDocument, position: Position): Promise<Hover | null> {
-    const provideXmlHovers = Config.get<boolean>('provideXmlHovers');
+    const provideXmlHovers = Config.get<boolean>('provideXmlHovers', true);
 
     if (!provideXmlHovers) {
       return null;

src/indexer/IndexManager.ts

@@ -20,6 +20,7 @@ import TemplateIndexer from './template/TemplateIndexer';
 import { TemplateIndexData } from './template/TemplateIndexData';
 import CronIndexer from './cron/CronIndexer';
 import { CronIndexData } from './cron/CronIndexData';
+import Config from 'common/Config';
 
 type IndexerInstance =
   | DiIndexer
@@ -41,7 +42,7 @@ type IndexerDataMap = {
 };
 
 class IndexManager {
-  private static readonly INDEX_BATCH_SIZE = 50;
+  private static readonly REPORTING_BATCH_SIZE = 500;
 
   protected indexers: IndexerInstance[] = [];
   protected indexStorage: IndexStorage;
@@ -77,40 +78,60 @@ class IndexManager {
     Logger.logWithTime('Indexing workspace', workspaceFolder.name);
 
     for (const indexer of this.indexers) {
-      this.indexStorage.loadIndex(workspaceFolder, indexer.getId(), indexer.getVersion());
-
-      if (!force && !this.shouldIndex(workspaceFolder, indexer)) {
-        Logger.logWithTime('Loaded index from storage', workspaceFolder.name, indexer.getId());
-        continue;
+      if (!force) {
+        Logger.logWithTime('Loading index from storage', workspaceFolder.name, indexer.getId());
+        this.indexStorage.loadIndex(workspaceFolder, indexer.getId(), indexer.getVersion());
+
+        if (!this.shouldIndex(workspaceFolder, indexer)) {
+          Logger.logWithTime('Loaded index from storage', workspaceFolder.name, indexer.getId());
+          continue;
+        }
       }
-      progress.report({ message: `Indexing - ${indexer.getName()}`, increment: 0 });
+      Logger.logWithTime('Indexing', indexer.getName());
 
-      const indexData = this.getIndexStorageData(indexer.getId()) || new Map();
+      progress.report({
+        message: `Indexing - ${indexer.getName()} [...]`,
+      });
 
-      Logger.logWithTime('Indexing', indexer.getName());
-      const files = await workspace.findFiles(indexer.getPattern(workspaceUri), 'dev/**');
+      const indexData = new Map();
+
+      Logger.logWithTime('Discovering files to index');
+
+      const files = await workspace.findFiles(
+        indexer.getPattern(workspaceUri),
+        indexer.getExcludePattern(workspaceUri)
+      );
 
       let doneCount = 0;
       const totalCount = files.length;
+      Logger.logWithTime('Found', totalCount, 'files to index');
 
-      for (let i = 0; i < files.length; i += IndexManager.INDEX_BATCH_SIZE) {
-        const batch = files.slice(i, i + IndexManager.INDEX_BATCH_SIZE);
+      const batchSize = Config.get<number>('magento-toolbox.indexingBatchSize', 25);
+
+      for (let i = 0; i < files.length; i += batchSize) {
+        const batch = files.slice(i, i + batchSize);
 
         await Promise.all(
           batch.map(async file => {
-            const data = await indexer.indexFile(file);
+            try {
+              const data = await indexer.indexFile(file);
 
-            if (data !== undefined) {
-              indexData.set(file.fsPath, data);
-            }
+              if (data !== undefined) {
+                indexData.set(file.fsPath, data);
+              }
 
-            doneCount++;
-            const pct = Math.round((doneCount / totalCount) * 100);
+              doneCount++;
 
-            progress.report({
-              message: `Indexing - ${indexer.getName()} [${doneCount}/${totalCount}]`,
-              increment: pct,
-            });
+              if (doneCount % IndexManager.REPORTING_BATCH_SIZE === 0) {
+                Logger.logWithTime('Indexed', doneCount, 'files of', totalCount);
+
+                progress.report({
+                  message: `Indexing - ${indexer.getName()} [${doneCount}/${totalCount}]`,
+                });
+              }
+            } catch (error) {
+              Logger.error('Error indexing file', file.fsPath, String(error));
+            }
           })
         );
       }
@@ -122,7 +143,7 @@ class IndexManager {
 
       Logger.logWithTime('Indexing', indexer.getName(), 'done');
 
-      progress.report({ increment: 100 });
+      progress.report({ message: `Indexing - ${indexer.getName()} [done]` });
     }
 
     Logger.logWithTime('Finished indexing workspace', workspaceFolder.name);

src/indexer/Indexer.ts

@@ -1,10 +1,15 @@
-import { type GlobPattern, type Uri } from 'vscode';
+import { type GlobPattern, type Uri, RelativePattern } from 'vscode';
 import { IndexerKey } from 'types/indexer';
 
 export abstract class Indexer<D = any> {
   public abstract getId(): IndexerKey;
   public abstract getName(): string;
   public abstract getPattern(uri: Uri): GlobPattern;
+
+  public getExcludePattern(uri: Uri): GlobPattern | null {
+    return new RelativePattern(uri, '{**/node_modules/**,**/dev/**,**/pub/**}');
+  }
+
   public abstract indexFile(uri: Uri): Promise<D | undefined>;
   public abstract getVersion(): number;
 }

src/util/Logger.ts

@@ -3,13 +3,18 @@ import { window } from 'vscode';
 export class Logger {
   private channel = window.createOutputChannel('Magento Toolbox');
 
-  public log(...message: string[]) {
+  public log(...message: (string | number)[]) {
     this.channel.appendLine(message.join(' '));
+    console.log('[Magento Toolbox]', ...message);
   }
 
-  public logWithTime(...message: string[]) {
+  public logWithTime(...message: (string | number)[]) {
     this.log(new Date().toISOString(), ...message);
   }
+
+  public error(...message: (string | number)[]) {
+    this.logWithTime('ERROR', ...message);
+  }
 }
 
 export default new Logger();