feat: remove unused imports

Author: manfredsteyer

libs/native-federation-core/package.json

@@ -1,11 +1,14 @@
 {
   "name": "@softarc/native-federation",
-  "version": "3.0.2",
+  "version": "3.1.0",
   "type": "commonjs",
   "license": "MIT",
   "dependencies": {
     "json5": "^2.2.0",
     "chalk": "^4.1.2",
-    "@softarc/native-federation-runtime": "3.0.2"
+    "@softarc/native-federation-runtime": "3.1.0"
+  },
+  "peerDependencies": {
+    "@softarc/sheriff-core": "^0.18.0"
   }
 }

libs/native-federation-core/src/lib/config/federation-config.ts

@@ -26,6 +26,7 @@ export interface FederationConfig {
   externals?: string[];
   features?: {
     mappingVersion?: boolean;
+    ignoreUnusedDeps?: boolean;
   };
 }
 
@@ -53,5 +54,6 @@ export interface NormalizedFederationConfig {
   externals: string[];
   features: {
     mappingVersion: boolean;
+    ignoreUnusedDeps: boolean;
   };
 }

libs/native-federation-core/src/lib/config/share-utils.ts

@@ -290,7 +290,14 @@ function getDefaultEntry(
   if (!entry) {
     entry = exports[key]?.['import'];
     if (typeof entry === 'object') {
-      entry = entry['import'];
+      entry = entry['import'] ?? entry['default'];
+    }
+  }
+
+  if (!entry) {
+    entry = exports[key]?.['require'];
+    if (typeof entry === 'object') {
+      entry = entry['require'] ?? entry['default'];
     }
   }
 

libs/native-federation-core/src/lib/config/with-native-federation.ts

@@ -27,6 +27,7 @@ export function withNativeFederation(
     externals: config.externals ?? [],
     features: {
       mappingVersion: config.features?.mappingVersion ?? false,
+      ignoreUnusedDeps: config.features?.ignoreUnusedDeps ?? false,
     },
   };
 }

libs/native-federation-core/src/lib/core/bundle-shared.ts

@@ -10,7 +10,6 @@ import { SharedInfo } from '@softarc/native-federation-runtime';
 import { FederationOptions } from './federation-options';
 import { copySrcMapIfExists } from '../utils/copy-src-map-if-exists';
 import { logger } from '../utils/logger';
-import { normalize } from '../utils/normalize';
 import crypto from 'crypto';
 import { DEFAULT_EXTERNAL_LIST } from './default-external-list';
 import { BuildResult } from './build-adapter';
@@ -77,9 +76,13 @@ export async function bundleShared(
       "Skip packages you don't want to share in your federation config"
     );
   }
+  
+  // If we build for the browser and don't remote unused deps from the shared config,
+  // we need to exclude typical node libs to avoid compilation issues
+  const useDefaultExternalList = platform === 'browser' && !config.features.ignoreUnusedDeps;
 
   const additionalExternals =
-    platform === 'browser' ? DEFAULT_EXTERNAL_LIST : [];
+      useDefaultExternalList ? DEFAULT_EXTERNAL_LIST : [];
 
   let bundleResult: BuildResult[] | null = null;
 

libs/native-federation-core/src/lib/core/federation-options.ts

@@ -7,4 +7,5 @@ export interface FederationOptions {
   dev?: boolean;
   watch?: boolean;
   packageJson?: string;
+  entryPoint?: string;
 }

libs/native-federation-core/src/lib/core/load-federation-config.ts

@@ -2,6 +2,7 @@ import { NormalizedFederationConfig } from '../config/federation-config';
 import { FederationOptions } from './federation-options';
 import * as path from 'path';
 import * as fs from 'fs';
+import { removeUnusedDeps } from './remove-unused-deps';
 
 export async function loadFederationConfig(
   fedOptions: FederationOptions
@@ -16,5 +17,18 @@ export async function loadFederationConfig(
   }
 
   const config = (await import(fullConfigPath)) as NormalizedFederationConfig;
+
+  if (config.features.ignoreUnusedDeps && !fedOptions.entryPoint) {
+    throw new Error(`The feature ignoreUnusedDeps needs the application's entry point. Please set it in your federation options!`);
+  }
+
+  if (config.features.ignoreUnusedDeps) {
+    return removeUnusedDeps(
+      config,
+      fedOptions.entryPoint ?? '',
+      fedOptions.workspaceRoot,
+    )
+  }
+
   return config;
 }

libs/native-federation-core/src/lib/core/remove-unused-deps.ts

@@ -0,0 +1,118 @@
+import { getProjectData } from '@softarc/sheriff-core';
+import path from 'path';
+import fs from 'fs';
+import { cwd } from 'process';
+import { NormalizedFederationConfig } from '../config/federation-config';
+import { getPackageInfo, PackageInfo } from '../utils/package-info';
+import { getExternalImports as extractExternalImports } from '../utils/get-external-imports';
+import { MappedPath } from '../utils/mapped-paths';
+import { ProjectData } from '@softarc/sheriff-core/src/lib/api/get-project-data';
+
+export function removeUnusedDeps(config: NormalizedFederationConfig, main: string, workspaceRoot: string): NormalizedFederationConfig {
+  const fileInfos = getProjectData(main, cwd(), {
+    includeExternalLibraries: true,
+  });
+
+  const usedDeps = findUsedDeps(fileInfos, workspaceRoot, config);
+  const usedPackageNames = usedDeps.usedPackageNames;
+  const usedMappings = usedDeps.usedMappings;
+
+  const usedPackageNamesWithTransient = addTransientDeps(usedPackageNames, workspaceRoot);
+  const filteredShared = filterShared(config, usedPackageNamesWithTransient);
+
+  return {
+    ...config,
+    shared: filteredShared,
+    sharedMappings: [...usedMappings],
+  };
+}
+
+function filterShared(config: NormalizedFederationConfig, usedPackageNamesWithTransient: Set<string>) {
+    const filteredSharedNames = Object.keys(config.shared).filter(
+        (shared) => usedPackageNamesWithTransient.has(shared)
+    );
+
+    const filteredShared = filteredSharedNames.reduce(
+        (acc, curr) => ({ ...acc, [curr]: config.shared[curr] }),
+        {}
+    );
+    return filteredShared;
+}
+
+function findUsedDeps(fileInfos: ProjectData, workspaceRoot: string, config: NormalizedFederationConfig) {
+    const usedPackageNames = new Set<string>();
+    const usedMappings = new Set<MappedPath>();
+
+    for (const fileName of Object.keys(fileInfos)) {
+        const fileInfo = fileInfos[fileName];
+
+        if (!fileInfo || !fileInfo.externalLibraries) {
+            continue;
+        }
+
+        for (const pckg of fileInfo.externalLibraries) {
+            usedPackageNames.add(pckg);
+        }
+
+        const fullFileName = path.join(workspaceRoot, fileName);
+        const mappings = config.sharedMappings.filter(
+            (sm) => sm.path === fullFileName
+        );
+
+        for (const mapping of mappings) {
+            usedMappings.add(mapping);
+        }
+    }
+    return { usedPackageNames, usedMappings };
+}
+
+function addTransientDeps(packages: Set<string>, workspaceRoot: string) {
+    const packagesAndPeers = new Set<string>([...packages]);
+    const discovered = new Set<string>(packagesAndPeers);
+    const stack = [...packagesAndPeers];
+
+    while (stack.length > 0) {
+        const dep = stack.pop();
+
+        if (!dep) {
+            continue;
+        }
+
+        const pInfo = getPackageInfo(dep, workspaceRoot);
+
+        if (!pInfo) {
+            continue;
+        }
+        
+        const peerDeps = getExternalImports(pInfo, workspaceRoot);
+
+        for (const peerDep of peerDeps) {
+            if (!discovered.has(peerDep)) {
+                discovered.add(peerDep);
+                stack.push(peerDep);
+                packagesAndPeers.add(peerDep);
+            }
+        }
+    }
+    return packagesAndPeers;
+}
+
+function getExternalImports(pInfo: PackageInfo, workspaceRoot: string) {
+    const encodedPackageName = pInfo.packageName.replace(/[^A-Za-z0-9]/g, '_');
+    const cacheFileName = `${encodedPackageName}-${pInfo.version}.deps.json`;
+    const cachePath = path.join(workspaceRoot, 'node_modules/.cache/native-federation');
+    const cacheFilePath = path.join(cachePath, cacheFileName);
+
+    const cacheHit = fs.existsSync(cacheFilePath);
+
+    let peerDeps;
+    if (cacheHit) {
+        peerDeps = JSON.parse(fs.readFileSync(cacheFilePath, 'utf-8'));
+    }
+    else {
+        peerDeps = extractExternalImports(pInfo.entryPoint);
+        fs.mkdirSync(cachePath, { recursive: true });
+        fs.writeFileSync(cacheFilePath, JSON.stringify(peerDeps, undefined, 2), 'utf-8');
+    }
+    return peerDeps;
+}

libs/native-federation-core/src/lib/utils/get-external-imports.ts

@@ -0,0 +1,101 @@
+import * as path from 'path';
+import * as fs from 'fs';
+import * as ts from 'typescript';
+
+export function getExternalImports(entryFilePath: string) {
+  const visited = new Set<string>();
+  const externals = new Set<string>();
+
+  function isExternal(specifier: string) {
+    return !specifier.startsWith('.') && !path.isAbsolute(specifier);
+  }
+
+  function normalizeExternal(specifier: string) {
+    return specifier;
+  }
+
+  function resolveAsFileOrDirectory(p: string) {
+    const abs = path.resolve(p);
+
+    if (fs.existsSync(abs) && fs.statSync(abs).isFile()) return abs;
+
+    const extensions = ['.ts', '.js', '.mjs', '.cjs'];
+    for (const ext of extensions) {
+      if (fs.existsSync(abs + ext) && fs.statSync(abs + ext).isFile()) {
+        return abs + ext;
+      }
+    }
+
+    if (fs.existsSync(abs) && fs.statSync(abs).isDirectory()) {
+      for (const file of extensions.map((e) => 'index' + e)) {
+        const indexPath = path.join(abs, file);
+        if (fs.existsSync(indexPath) && fs.statSync(indexPath).isFile()) {
+          return indexPath;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  function visit(filePath: string) {
+    const absPath = path.resolve(filePath);
+    if (visited.has(absPath)) return;
+    visited.add(absPath);
+
+    const resolved = resolveAsFileOrDirectory(absPath);
+    if (!resolved) return;
+
+    const source = fs.readFileSync(resolved, 'utf8');
+    const sourceFile = ts.createSourceFile(
+      resolved,
+      source,
+      ts.ScriptTarget.Latest,
+      true
+    );
+
+    function walk(node: ts.Node) {
+      if (ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) {
+        const moduleSpecifier = node.moduleSpecifier;
+        if (moduleSpecifier && ts.isStringLiteral(moduleSpecifier)) {
+          const spec = moduleSpecifier.text;
+          if (isExternal(spec)) {
+            externals.add(normalizeExternal(spec));
+          } else {
+            const resolvedPath = resolveAsFileOrDirectory(
+              path.resolve(path.dirname(resolved ?? ''), spec)
+            );
+            if (resolvedPath) visit(resolvedPath);
+          }
+        }
+      }
+
+      if (
+        ts.isCallExpression(node) &&
+        ts.isIdentifier(node.expression) &&
+        node.expression.kind === ts.SyntaxKind.Identifier &&
+        node.expression.escapedText === 'require' &&
+        node.arguments.length === 1 &&
+        ts.isStringLiteral(node.arguments[0])
+      ) {
+        const spec = node.arguments[0].text;
+        if (isExternal(spec)) {
+          externals.add(normalizeExternal(spec));
+        } else {
+          const resolvedPath = resolveAsFileOrDirectory(
+            path.resolve(path.dirname(resolved ?? ''), spec)
+          );
+          if (resolvedPath) visit(resolvedPath);
+        }
+      }
+
+      ts.forEachChild(node, walk);
+    }
+
+    ts.forEachChild(sourceFile, walk);
+  }
+
+  visit(entryFilePath);
+
+  return Array.from(externals);
+}

libs/native-federation-esbuild/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@softarc/native-federation-esbuild",
-  "version": "3.0.2",
+  "version": "3.1.0",
   "type": "commonjs",
   "dependencies": {
     "@rollup/plugin-commonjs": "^22.0.2",