perf(dts-plugin): add fetch timeout and add log (#3554)

Author: 2heal1

.changeset/seven-geckos-carry.md

@@ -0,0 +1,6 @@
+---
+'@module-federation/dts-plugin': patch
+'@module-federation/sdk': patch
+---
+
+perf(dts-plugin): add fetch timeout and add log

packages/dts-plugin/src/core/configurations/hostPlugin.test.ts

@@ -42,6 +42,7 @@ describe('hostPlugin', () => {
           consumeAPITypes: false,
           runtimePkgs: [],
           remoteTypeUrls: {},
+          timeout: 60000,
         });
 
         expect(mapRemotesToDownload).toStrictEqual({
@@ -68,6 +69,7 @@ describe('hostPlugin', () => {
           consumeAPITypes: false,
           runtimePkgs: [],
           remoteTypeUrls: {},
+          timeout: 60000,
         };
 
         const { hostOptions, mapRemotesToDownload } =

packages/dts-plugin/src/core/configurations/hostPlugin.ts

@@ -18,6 +18,7 @@ const defaultOptions = {
   consumeAPITypes: false,
   runtimePkgs: [],
   remoteTypeUrls: {},
+  timeout: 60000,
 } satisfies Partial<HostOptions>;
 
 const buildZipUrl = (hostOptions: Required<HostOptions>, url: string) => {

packages/dts-plugin/src/core/lib/DTSManager.general.spec.ts

@@ -154,8 +154,8 @@ describe('DTSManager General Tests', () => {
         url: 'http://example.com/remote.manifest.json',
         alias: 'test-alias',
       };
-
-      const result = await dtsManager.requestRemoteManifest(remoteInfo);
+      // @ts-expect-error only need timeout, which is not required
+      const result = await dtsManager.requestRemoteManifest(remoteInfo, {});
       expect(result.zipUrl).toBeDefined();
       expect(result.apiTypeUrl).toBeDefined();
       expect(result.zipUrl).toContain('http://example.com/types.zip');
@@ -181,7 +181,8 @@ describe('DTSManager General Tests', () => {
         alias: 'test-alias',
       };
 
-      const result = await dtsManager.requestRemoteManifest(remoteInfo);
+      // @ts-expect-error only need timeout, which is not required
+      const result = await dtsManager.requestRemoteManifest(remoteInfo, {});
       expect(result.zipUrl).toBeDefined();
       expect(result.apiTypeUrl).toBe('');
     });
@@ -195,7 +196,8 @@ describe('DTSManager General Tests', () => {
         alias: 'test-alias',
       };
 
-      const result = await dtsManager.requestRemoteManifest(remoteInfo);
+      // @ts-expect-error only need timeout, which is not required
+      const result = await dtsManager.requestRemoteManifest(remoteInfo, {});
       expect(result).toEqual(remoteInfo);
     });
 
@@ -220,7 +222,8 @@ describe('DTSManager General Tests', () => {
         alias: 'test-alias',
       };
 
-      const result = await dtsManager.requestRemoteManifest(remoteInfo);
+      // @ts-expect-error only need timeout, which is not required
+      const result = await dtsManager.requestRemoteManifest(remoteInfo, {});
       expect(result.zipUrl).toContain('http://example.com/custom/types.zip');
       expect(result.apiTypeUrl).toContain('http://example.com/custom/api.d.ts');
     });
@@ -242,6 +245,7 @@ describe('DTSManager General Tests', () => {
       abortOnError: false,
       consumeAPITypes: true,
       maxRetries: 3,
+      timeout: 60000,
     };
 
     it('should successfully download types archive', async () => {
@@ -347,7 +351,8 @@ describe('DTSManager General Tests', () => {
       vi.mocked(axios.get).mockResolvedValueOnce({ data: apiTypeContent });
       vi.spyOn(fs, 'writeFileSync');
 
-      await dtsManager.downloadAPITypes(remoteInfo, '/tmp/types');
+      // @ts-expect-error only need timeout, which is not required
+      await dtsManager.downloadAPITypes(remoteInfo, '/tmp/types', {});
 
       expect(fs.writeFileSync).toHaveBeenCalled();
       const writeCall = vi.mocked(fs.writeFileSync).mock.calls[0];

packages/dts-plugin/src/core/lib/DTSManager.ts

@@ -205,6 +205,7 @@ class DTSManager {
 
   async requestRemoteManifest(
     remoteInfo: RemoteInfo,
+    hostOptions: Required<HostOptions>,
   ): Promise<Required<RemoteInfo>> {
     try {
       if (!remoteInfo.url.includes(MANIFEST_EXT)) {
@@ -214,7 +215,7 @@ class DTSManager {
         return remoteInfo as Required<RemoteInfo>;
       }
       const url = remoteInfo.url;
-      const res = await axiosGet(url);
+      const res = await axiosGet(url, { timeout: hostOptions.timeout });
       const manifestJson = res.data as unknown as Manifest;
       if (!manifestJson.metaData.types.zip) {
         throw new Error(`Can not get ${remoteInfo.name}'s types archive url!`);
@@ -280,14 +281,15 @@ class DTSManager {
   async downloadAPITypes(
     remoteInfo: Required<RemoteInfo>,
     destinationPath: string,
+    hostOptions: Required<HostOptions>,
   ) {
     const { apiTypeUrl } = remoteInfo;
     if (!apiTypeUrl) {
       return;
     }
     try {
       const url = apiTypeUrl;
-      const res = await axiosGet(url);
+      const res = await axiosGet(url, { timeout: hostOptions.timeout });
       let apiTypeFile = res.data as string;
       apiTypeFile = apiTypeFile.replaceAll(
         REMOTE_ALIAS_IDENTIFIER,
@@ -389,8 +391,10 @@ class DTSManager {
       async (item) => {
         const remoteInfo = item[1];
         if (!this.remoteAliasMap[remoteInfo.alias]) {
-          const requiredRemoteInfo =
-            await this.requestRemoteManifest(remoteInfo);
+          const requiredRemoteInfo = await this.requestRemoteManifest(
+            remoteInfo,
+            hostOptions,
+          );
           this.remoteAliasMap[remoteInfo.alias] = requiredRemoteInfo;
         }
 
@@ -434,7 +438,11 @@ class DTSManager {
               return;
             }
 
-            await this.downloadAPITypes(remoteInfo, destinationPath);
+            await this.downloadAPITypes(
+              remoteInfo,
+              destinationPath,
+              hostOptions,
+            );
           }),
         );
         this.consumeAPITypes(hostOptions);
@@ -507,6 +515,7 @@ class DTSManager {
           const addNew = await this.downloadAPITypes(
             requiredRemoteInfo,
             destinationPath,
+            hostOptions,
           );
           if (addNew) {
             this.consumeAPITypes(hostOptions);
@@ -531,8 +540,10 @@ class DTSManager {
           );
           if (remoteInfo) {
             if (!this.remoteAliasMap[remoteInfo.alias]) {
-              const requiredRemoteInfo =
-                await this.requestRemoteManifest(remoteInfo);
+              const requiredRemoteInfo = await this.requestRemoteManifest(
+                remoteInfo,
+                hostOptions,
+              );
               this.remoteAliasMap[remoteInfo.alias] = requiredRemoteInfo;
             }
             await consumeTypes(this.remoteAliasMap[remoteInfo.alias]);
@@ -550,7 +561,7 @@ class DTSManager {
               });
               fileLog(`start request manifest`, 'consumeTypes', 'info');
               this.updatedRemoteInfos[updatedRemoteInfo.name] =
-                await this.requestRemoteManifest(parsedRemoteInfo);
+                await this.requestRemoteManifest(parsedRemoteInfo, hostOptions);
               fileLog(
                 `end request manifest, this.updatedRemoteInfos[updatedRemoteInfo.name]: ${JSON.stringify(
                   this.updatedRemoteInfos[updatedRemoteInfo.name],

packages/dts-plugin/src/core/lib/archiveHandler.ts

@@ -62,6 +62,7 @@ export const downloadTypesArchive = (hostOptions: Required<HostOptions>) => {
         const url = fileToDownload;
         const response = await axiosGet(url, {
           responseType: 'arraybuffer',
+          timeout: hostOptions.timeout,
         }).catch(downloadErrorLogger(destinationFolder, url));
         if (
           typeof response.headers?.['content-type'] === 'string' &&

packages/dts-plugin/src/core/lib/utils.ts

@@ -154,5 +154,6 @@ export async function axiosGet(url: string, config?: AxiosRequestConfig) {
       headers: getEnvHeaders(),
     },
     ...config,
+    timeout: config?.timeout || 60000,
   });
 }

packages/dts-plugin/src/plugins/ConsumeTypesPlugin.ts

@@ -1,11 +1,13 @@
-import type { Compiler, WebpackPluginInstance } from 'webpack';
+import { logger } from '@module-federation/sdk';
 import {
   normalizeOptions,
   type moduleFederationPlugin,
 } from '@module-federation/sdk';
 import { validateOptions, consumeTypes } from '../core/index';
 import { isPrd } from './utils';
 
+import type { Compiler, WebpackPluginInstance } from 'webpack';
+
 export class ConsumeTypesPlugin implements WebpackPluginInstance {
   pluginOptions: moduleFederationPlugin.ModuleFederationPluginOptions;
   dtsOptions: moduleFederationPlugin.PluginDtsOptions;
@@ -70,6 +72,7 @@ export class ConsumeTypesPlugin implements WebpackPluginInstance {
       typeof normalizedConsumeTypes.remoteTypeUrls === 'function'
         ? normalizedConsumeTypes.remoteTypeUrls()
         : Promise.resolve(normalizedConsumeTypes.remoteTypeUrls);
+    logger.debug('start fetching remote types...');
     const promise = fetchRemoteTypeUrlsPromise.then((remoteTypeUrls) => {
       consumeTypes({
         ...finalOptions,
@@ -97,6 +100,7 @@ export class ConsumeTypesPlugin implements WebpackPluginInstance {
         async () => {
           // await consume types promise to make sure the consumer not throw types error
           await promise;
+          logger.debug('fetch remote types success!');
         },
       );
     });

packages/dts-plugin/src/plugins/GenerateTypesPlugin.ts

@@ -1,8 +1,8 @@
-import type { Compilation, Compiler, WebpackPluginInstance } from 'webpack';
 import fs from 'fs';
 import path from 'path';
 import { isDev, getCompilerOutputDir } from './utils';
 import {
+  logger,
   normalizeOptions,
   type moduleFederationPlugin,
 } from '@module-federation/sdk';
@@ -14,6 +14,8 @@ import {
   type DTSManagerOptions,
 } from '../core/index';
 
+import type { Compilation, Compiler, WebpackPluginInstance } from 'webpack';
+
 export class GenerateTypesPlugin implements WebpackPluginInstance {
   pluginOptions: moduleFederationPlugin.ModuleFederationPluginOptions;
   dtsOptions: moduleFederationPlugin.PluginDtsOptions;
@@ -119,7 +121,9 @@ export class GenerateTypesPlugin implements WebpackPluginInstance {
           return;
         }
 
+        logger.debug('start generating types...');
         await generateTypesFn(finalOptions);
+        logger.debug('generate types success!');
         const config = finalOptions.remote.moduleFederationConfig;
         let zipPrefix = '';
         if (typeof config.manifest === 'object' && config.manifest.filePath) {
@@ -229,6 +233,7 @@ export class GenerateTypesPlugin implements WebpackPluginInstance {
         if (finalOptions.displayErrorInTerminal) {
           console.error('Error in mf:generateTypes processAssets hook:', err);
         }
+        logger.debug('generate types fail!');
       }
     };
 

packages/sdk/src/types/plugins/ModuleFederationPlugin.ts

@@ -154,6 +154,7 @@ export interface DtsHostOptions {
   consumeAPITypes?: boolean;
   runtimePkgs?: string[];
   remoteTypeUrls?: (() => Promise<RemoteTypeUrls>) | RemoteTypeUrls;
+  timeout?: number;
 }
 
 export interface DtsRemoteOptions {