docs: Create README and doc comments for patch generator

Author: clemyan

packages/plugin-compat/extra/PatchGenerator.ts

@@ -25,7 +25,19 @@ export abstract class PatchGenerator<S extends {id: string, range: string}> {
     this.patches = ppath.join(npath.toPortablePath(__dirname), this.name as Filename, `patches`);
   }
 
+  /**
+   * Given the path to the build cache directory, populate it by saving the
+   * "before" state of the slice into `<path>/base` and the "after" state into
+   * `<path>/patched`.
+   *
+   * The build cache directory is guaranteed to exists, but the `base` and
+   * `patched` subdirectories are not.
+   */
   protected abstract build(slice: S, path: PortablePath): Promise<void>;
+  /**
+   * Given a slice, return the list of package versions the generated patch
+   * should be validated against.
+   */
   protected abstract getValidateVersions(slice: S): Promise<Array<string>>;
 
   private async fetchTarball(version: string): Promise<Buffer> {
@@ -38,6 +50,10 @@ export abstract class PatchGenerator<S extends {id: string, range: string}> {
 
     return Buffer.from(await response.arrayBuffer());
   }
+  /**
+   * Return the tarball for the given version of the package with caching. If
+   * not already cached, it will be fetched from the npm registry.
+   */
   protected async getTarball(version: string): Promise<Buffer> {
     const path = ppath.join(this.tmp, `tarballs`, `${version}.tgz` as Filename);
     if (await xfs.existsPromise(path))
@@ -52,6 +68,10 @@ export abstract class PatchGenerator<S extends {id: string, range: string}> {
     return tarball;
   }
 
+  /**
+   * Generate a unified diff between the `base` and `patched` sub-directories of
+   * the given directory, with the custom semver exclusivity header.
+   */
   protected async diff(range: string, dir: PortablePath): Promise<string> {
     const patch = await spawn(`git`, [
       `diff`,
@@ -89,6 +109,10 @@ export abstract class PatchGenerator<S extends {id: string, range: string}> {
       });
   }
 
+  /**
+   * Create the patch for the given slice, reusing any existing cached patch or
+   * build if available, and write it to disk if not already cached.
+   */
   protected createPatch(slice: S): Promise<{path: PortablePath, content: string}> {
     return logger.section(`Create patch`, async () => {
       const path = ppath.join(this.patches, `patch-${slice.id}.diff` as Filename);
@@ -128,8 +152,8 @@ export abstract class PatchGenerator<S extends {id: string, range: string}> {
     });
   }
 
-  protected readonly envs = new Map<string, Promise<PortablePath>>();
-  protected async prepareValidationEnv(version: string): Promise<PortablePath> {
+  private readonly envs = new Map<string, Promise<PortablePath>>();
+  private async prepareValidationEnv(version: string): Promise<PortablePath> {
     const path = ppath.join(this.tmp, `validate`, version as Filename);
     const [tarball] = await Promise.all([
       this.getTarball(version),
@@ -138,6 +162,10 @@ export abstract class PatchGenerator<S extends {id: string, range: string}> {
     await tgzUtils.extractArchiveTo(tarball, new CwdFS(path), {stripComponents: 1});
     return path;
   }
+  /**
+   * Prepare the validation environment for the specified version by extracting
+   * the package tarball into a temporary directory.
+   */
   protected async getValidationEnv(version: string): Promise<PortablePath> {
     return miscUtils.getFactoryWithDefault(this.envs, version, () =>  this.prepareValidationEnv(version));
   }
@@ -158,6 +186,10 @@ export abstract class PatchGenerator<S extends {id: string, range: string}> {
         .catch(() => {}); // Prevent unhandled rejection - errors will be handled during validation
     }
   }
+  /**
+   * Validate that the given patch can be cleanly applied to the versions of the
+   * package as selected by {@link getValidateVersions}.
+   */
   protected validatePatch(slice: S, patch: string): Promise<void> {
     return logger.section(`Validate patch`, async () => {
       const versions = await this.getValidateVersions(slice);
@@ -175,6 +207,9 @@ export abstract class PatchGenerator<S extends {id: string, range: string}> {
     });
   }
 
+  /**
+   * Create, write to disk, and validate the patch for the given slice.
+   */
   protected async generatePatch(slice: S): Promise<string> {
     const clearBuildCache = () => xfs.removeSync(ppath.join(this.tmp, `builds`, slice.id as Filename));
 
@@ -198,6 +233,11 @@ export abstract class PatchGenerator<S extends {id: string, range: string}> {
     });
   }
 
+  /**
+   * Generate all patches and write the compressed TS bundle to the specified
+   * path. If one or more ranges are specified, caches are not used for slices
+   * whose range overlaps with any of the specified ranges.
+   */
   public async generateBundle(ranges: Array<string>, path: PortablePath): Promise<void> {
     // Start preparing validation environments immediately
     const controller = new AbortController();

packages/plugin-compat/extra/README.md

@@ -0,0 +1,49 @@
+# Compat patches generator
+
+## Overview
+
+This directory contains scripts and files to generate the builtin patches. Each patched package has its own sub-directory.
+
+For each package, the `patches` sub-directory contains the set of builtin patches that are applied to it. Each of those patch files is also known as a patch *slice*, and is marked with a semver range of package versions that the patch applies to.
+
+The `gen-<package>-patch.ts` script is used to (re-)generate those patches and update the files in `../source/patches`, which are then bundled with Yarn.
+
+## Running
+
+The generator scripts can be run via the package script `build:patch:<package>`.
+
+As a high-level overview, the script generates each patch slice by:
+1. Running a package-specific build process to create the "before" and "after" states of the package
+2. Running `git diff` to calculate the diff between the two
+3. Validating the diff by trying to apply it as a patch to some versions of the package
+4. Saving it to `patches`
+
+After all slices are generated, they are aggregated, compressed, and encoded into the `../source/patches` bundles.
+
+## Caching
+
+There are two layers of caching applied. First, if a patch already exists in the `patches` directory, then the diff is used instead of building and calculating a fresh diff. Second, the "before" and "after" states created during builds are saved and reused to skip future builds.
+
+The cache and other temporary files used by the generator are store in `<tmp>/yarn-compat-gen-patches/`, where `<tmp>` is the system temporary directory. This directory can be changed by setting `GEN_PATCHES_BASE_DIR` environment variable to an absolute path. The caches for each package are isolated under sub-directories.
+
+When running the generator scripts, one or more semver ranges can be given as additional arguments. In that case, when generating any slice whose range overlaps with the given ranges, the cached patches and builds are not reused and a fresh patch is generated.
+
+On Windows, it is recommended to exclude the cache from Windows Defender as its real-time protection has a huge negative impact to performance during I/O intensive builds.
+
+## Package-specific notes
+
+### `fsevents`
+
+The build process for `fsevents` is simply downloading and extracting the package from npm and copying patched and/or new files into the package.
+
+### `resolve`
+
+The build process for `resolve` is similar to that for `fsevents` -- downloading the package from npm and copying patched and/or new files into the package.
+
+### `typescript`
+
+For `typescript`, the build process is much heavier. We clone `yarnpkg/TypeScript` (our own fork of the TypeScript repo) from GitHub, cherry-pick commits that implement PnP, and build the TypeScript distributables.
+
+As the TypeScript repo uses Volta to pin the node and npm versions used to build it, installing Volta is recommended to ensure consistent builds. If Volta is not installed, the generator script only switches the npm version by running npm via Corepack. A warning is printed in this case.
+
+If you already have a local clone of `yarnpkg/TypeScript`, you can use git's alternates mechanism to allow git to find objects in the local clone, via the `GIT_ALTERNATE_OBJECT_DIRECTORIES` environment variable. This way, you can generate patches using commits in the local clone. The local clone is also used to speed up the script's cloning and fetching operations. However, do note that PRs that update the patch should only reference public commits in `yarnpkg/Typescript` or our CI checks will fail on the PR.