fix: improve sass compile speed by preloading files (#6881)

Author: CompuIves

packages/app/src/sandbox/eval/transpilers/sass/worker/resolver.ts

@@ -45,9 +45,10 @@ export function getPossibleSassPaths(
   return directories.map(directory => pathUtils.join(directory, filepath));
 }
 
-async function resolveAsyncModule(opts: {
+export async function resolveAsyncModule(opts: {
   path: string;
   options: {
+    isAbsolute: boolean;
     ignoredExtensions: typeof POTENTIAL_EXTENSIONS;
   };
   loaderContextId: number;
@@ -91,6 +92,7 @@ function existsPromise(opts: {
           const resolvedModule = await resolveAsyncModule({
             path: filepath,
             options: {
+              isAbsolute: filepath.startsWith('/'),
               ignoredExtensions: POTENTIAL_EXTENSIONS,
             },
             loaderContextId,
@@ -103,6 +105,19 @@ function existsPromise(opts: {
             return;
           }
 
+          try {
+            fs.mkdirSync(pathUtils.dirname(resolvedModule.path), {
+              recursive: true,
+            });
+          } catch (e) {
+            /* noop */
+          }
+          try {
+            fs.writeFileSync(resolvedModule.path, resolvedModule.code);
+          } catch (e) {
+            /* noop */
+          }
+
           r(resolvedModule.path);
         } catch (e) {
           r(false);
@@ -114,43 +129,6 @@ function existsPromise(opts: {
   });
 }
 
-/**
- * Return and stop as soon as one promise returns a truthy value
- */
-function firstTrue<T>(promises: Array<Promise<T | false>>): Promise<T> {
-  if (!promises.length) {
-    return Promise.reject(new Error('No promises supplied'));
-  }
-
-  return new Promise((resolvePromise, rejectPromise) => {
-    let resolved: boolean = false;
-    let firstError: any;
-
-    Promise.all(
-      promises.map(promise =>
-        promise.then(
-          result => {
-            if (result && !resolved) {
-              resolved = true;
-              resolvePromise(result);
-            }
-          },
-          err => {
-            firstError = firstError || err;
-          }
-        )
-      )
-    ).then(() => {
-      if (!resolved) {
-        const err =
-          firstError ||
-          new Error('None of the promises returned a truthy value');
-        rejectPromise(err);
-      }
-    });
-  });
-}
-
 async function resolvePotentialPath(opts: {
   fs: any;
   potentialPath: string;
@@ -163,17 +141,36 @@ async function resolvePotentialPath(opts: {
     const pathVariations = getPathVariations(
       pathUtils.basename(potentialPath)
     ).map(variation => pathUtils.join(pathDirName, variation));
-    const foundFilePath = await firstTrue<string>(
-      pathVariations.map(path =>
-        existsPromise({ fs, filepath: path, loaderContextId, childHandler })
-      )
-    );
-
-    if (!foundFilePath) {
-      return null;
+
+    // Try the first one first, that's often the right one and this way we don't spam
+    // the main thread.
+    const firstPath = pathVariations.shift();
+    const firstFoundFilePath = await existsPromise({
+      fs,
+      filepath: firstPath,
+      loaderContextId,
+      childHandler,
+    });
+
+    if (firstFoundFilePath) {
+      return firstPath;
+    }
+
+    for (const path of pathVariations) {
+      // eslint-disable-next-line no-await-in-loop
+      const result = await existsPromise({
+        fs,
+        filepath: path,
+        loaderContextId,
+        childHandler,
+      });
+
+      if (result) {
+        return path;
+      }
     }
 
-    return foundFilePath;
+    return null;
   } catch (err) {
     return null;
   }
@@ -198,7 +195,7 @@ interface ISassResolverOptions {
   includePaths?: Array<string>;
   env?: any;
   fs: any;
-  resolutionCache: { [k: string]: string };
+  resolutionCache: { [k: string]: Promise<string> };
   loaderContextId: number;
   childHandler: ChildHandler;
 }
@@ -249,15 +246,16 @@ export async function resolveSassUrl(opts: ISassResolverOptions) {
       return resolutionCache[potentialPath];
     }
 
-    // eslint-disable-next-line no-await-in-loop
-    const resolvedPath = await resolvePotentialPath({
+    resolutionCache[potentialPath] = resolvePotentialPath({
       fs,
       potentialPath,
       loaderContextId,
       childHandler,
     });
+
+    // eslint-disable-next-line no-await-in-loop
+    const resolvedPath = await resolutionCache[potentialPath];
     if (resolvedPath) {
-      resolutionCache[potentialPath] = resolvedPath;
       return resolvedPath;
     }
   }

packages/app/src/sandbox/eval/transpilers/sass/worker/sass-worker.ts

@@ -1,7 +1,8 @@
 import delay from '@codesandbox/common/lib/utils/delay';
 
-import { resolveSassUrl } from './resolver';
+import { resolveAsyncModule, resolveSassUrl } from './resolver';
 import { ChildHandler } from '../../worker-transpiler/child-handler';
+import { getModulesFromMainThread } from '../../utils/fs';
 
 // @ts-ignore
 self.window = self;
@@ -16,6 +17,7 @@ async function fetchSassLibrary() {
 }
 
 let fsInitialized = false;
+let fsLoading = false;
 const childHandler = new ChildHandler('sass-worker');
 
 interface ISassCompileOptions {
@@ -25,8 +27,10 @@ interface ISassCompileOptions {
   loaderContextId: number;
 }
 
-async function initFS() {
-  if (!fsInitialized) {
+async function initFS(loaderContextId: number) {
+  if (!fsLoading && !fsInitialized) {
+    await initializeBrowserFS(loaderContextId);
+  } else if (!fsInitialized) {
     while (!fsInitialized) {
       await delay(50); // eslint-disable-line
     }
@@ -38,7 +42,7 @@ async function compileSass(opts: ISassCompileOptions) {
 
   const Sass = await fetchSassLibrary();
 
-  await initFS();
+  await initFS(loaderContextId);
 
   // @ts-ignore
   // eslint-disable-next-line
@@ -54,8 +58,24 @@ async function compileSass(opts: ISassCompileOptions) {
     if (!foundFileCache[filepath]) {
       foundFileCache[filepath] = new Promise(
         (promiseResolve, promiseReject) => {
-          fs.readFile(filepath, {}, (error, data) => {
+          fs.readFile(filepath, {}, async (error, data) => {
             if (error) {
+              // Try to download it
+              const module = await resolveAsyncModule({
+                path: filepath,
+                loaderContextId: opts.loaderContextId,
+                childHandler,
+                options: {
+                  isAbsolute: false,
+                  ignoredExtensions: ['.sass', '.css', '.scss'],
+                },
+              });
+
+              if (module) {
+                promiseResolve(module.code);
+                return;
+              }
+
               promiseReject(error);
               return;
             }
@@ -146,24 +166,56 @@ async function compileSass(opts: ISassCompileOptions) {
   };
 }
 
-async function initializeBrowserFS() {
-  await new Promise(res => {
+async function initializeBrowserFS(loaderContextId: number) {
+  fsLoading = true;
+
+  const modules = await getModulesFromMainThread({
+    childHandler,
+    loaderContextId,
+  });
+
+  const tModules = {};
+  modules.forEach(module => {
+    tModules[module.path] = { module };
+  });
+
+  const bfsWrapper = {
+    getTranspiledModules: () => tModules,
+    addModule: () => {},
+    removeModule: () => {},
+    moveModule: () => {},
+    updateModule: () => {},
+  };
+
+  return new Promise(resolvePromise => {
     // @ts-ignore
-    // eslint-disable-next-line
     BrowserFS.configure(
       {
-        fs: 'WorkerFS',
-        options: { worker: self },
+        fs: 'OverlayFS',
+        options: {
+          writable: { fs: 'InMemory' },
+          readable: {
+            fs: 'CodeSandboxFS',
+            options: {
+              manager: bfsWrapper,
+            },
+          },
+        },
       },
-      () => {
-        res(null);
+      err => {
+        if (err) {
+          console.error(err);
+          return;
+        }
+        fsLoading = false;
+        fsInitialized = true;
+        resolvePromise(null);
+        // BrowserFS is initialized and ready-to-use!
       }
     );
   });
-
-  fsInitialized = true;
 }
 
 childHandler.registerFunction('compile', compileSass);
-childHandler.registerFSInitializer(initializeBrowserFS);
+childHandler.registerFSInitializer(() => {});
 childHandler.emitReady();

packages/sandpack-core/src/npm/dynamic/fetch-npm-module.ts

@@ -40,6 +40,7 @@ function prependRootPath(meta: Meta, rootPath: string): Meta {
 export interface FetchProtocol {
   file(name: string, version: string, path: string): Promise<string>;
   meta(name: string, version: string): Promise<Meta>;
+  massFiles?(name: string, version: string): Promise<Array<Module>>;
 }
 
 export function setCombinedMetas(givenCombinedMetas: Meta) {
@@ -94,6 +95,33 @@ function getMeta(
   }));
 }
 
+const downloadedAllDependencies = new Set<string>();
+/**
+ * If the protocol supports it, download all all files of the dependency
+ * at once. It's an optimization.
+ */
+export async function downloadAllDependencyFiles(
+  name: string,
+  version: string
+): Promise<Module[] | null> {
+  if (downloadedAllDependencies.has(`${name}@${version}`)) {
+    return null;
+  }
+
+  downloadedAllDependencies.add(`${name}@${version}`);
+
+  const [depName, depVersion] = resolveNPMAlias(name, version);
+  const nameWithoutAlias = depName.replace(ALIAS_REGEX, '');
+  const protocol = getFetchProtocol(depName, depVersion);
+
+  if (protocol.massFiles) {
+    // If the protocol supports returning many files at once, we opt for that instead.
+    return protocol.massFiles(nameWithoutAlias, depVersion);
+  }
+
+  return null;
+}
+
 export function downloadDependency(
   name: string,
   version: string,

packages/sandpack-core/src/npm/dynamic/fetch-protocols/npm-registry.ts

@@ -1,4 +1,5 @@
 import { satisfies, valid } from 'semver';
+import { Module } from '../../../types/module';
 import { FetchProtocol, Meta } from '../fetch-npm-module';
 import { fetchWithRetries } from './utils';
 import { TarStore } from './utils/tar-store';
@@ -235,6 +236,17 @@ export class NpmRegistryFetcher implements FetchProtocol {
     return this.tarStore.meta(name, tarball, this.getRequestInit());
   }
 
+  public async massFiles(name: string, version: string): Promise<Module[]> {
+    const versionInfo = await this.getVersionInfo(name, version);
+
+    const tarball = this.getTarballUrl(
+      name,
+      versionInfo.version,
+      versionInfo.dist.tarball
+    );
+    return this.tarStore.massFiles(name, tarball, this.getRequestInit());
+  }
+
   public condition = (name: string, version: string): boolean => {
     if (this.scopeWhitelist) {
       return this.scopeWhitelist.some(scope => {

packages/sandpack-core/src/npm/dynamic/fetch-protocols/utils/tar-store.ts

@@ -1,6 +1,7 @@
 import untar, { UntarredFiles } from 'isomorphic-untar-gzip';
+import { Module } from '../../../../types/module';
 
-import { Meta } from '../../fetch-npm-module';
+import { FetchProtocol, Meta } from '../../fetch-npm-module';
 import { fetchWithRetries } from '../utils';
 
 type DeserializedFetchedTar = {
@@ -11,7 +12,7 @@ type DeserializedFetchedTar = {
  * Responsible for fetching, caching and converting tars to a structure that sandpack
  * understands and can use
  */
-export class TarStore {
+export class TarStore implements FetchProtocol {
   private fetchedTars: {
     [key: string]: Promise<{ [path: string]: DeserializedFetchedTar }>;
   } = {};
@@ -77,4 +78,20 @@ export class TarStore {
 
     return meta;
   }
+
+  async massFiles(
+    name: string,
+    url: string,
+    requestInit?: RequestInit
+  ): Promise<Module[]> {
+    const tarKey = this.generateKey(name, url);
+    const tar = await (this.fetchedTars[tarKey] ||
+      this.fetchTar(name, url, requestInit));
+
+    return Object.keys(tar).map(path => ({
+      path,
+      code: tar[path].content,
+      downloaded: true,
+    }));
+  }
 }