feat(mono-pub)!: ADR-1 split prepare into prepareAll and prepareSingle (#30)

* feat(mono-pub): add getExecutionOrder util

* feat(mono-pub)!: ADR-1 split prepare into prepareAll and prepareSingle

BREAKING-CHANGE: plugin chain behaviour change, prepare step is removed and replaced by prepareSingle and prepareAll

* refactor: changed docs and fixed tests

* fix: PR review fixes

Author: SavelevMatthew

bin/publish.js

@@ -4,11 +4,16 @@ const git = require('@mono-pub/git')
 const github = require('@mono-pub/github')
 const npm = require('@mono-pub/npm')
 const commitAnalyzer = require('@mono-pub/commit-analyzer')
+const { getExecutionOrder } = require('mono-pub/utils')
 
+/** @type {import('mono-pub').MonoPubPlugin} */
 const builder = {
     name: '@mono-pub/local-builder',
-    async prepare(_, ctx) {
-        await execa('yarn', ['build'], { cwd: ctx.cwd })
+    async prepareAll({ foundPackages }, ctx) {
+        const batches = getExecutionOrder(foundPackages, { batching: true })
+        for (const batch of batches) {
+            await execa('yarn', ['build', ...batch.map((pkg) => `--filter=${pkg.name}`)], { cwd: ctx.cwd })
+        }
     },
 }
 

packages/mono-pub/README.md

@@ -67,15 +67,15 @@ The mono-pub plugins handle this out of the box (See [`@mono-pub/github`](https:
 We divide the publishing process into several steps, 
 allowing you to control most of the process yourself through pre-made plugins or using your own.
 
-| Step             | Description                                                                                                                                              |
-|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Setup            | Sets up the plugin, checks all the conditions necessary for work.                                                                                        |
-| Get Last Release | Obtains latest released version for each package (Usually from tags analysis)                                                                            |
-| Extract commits  | Extracts commits relevant to specific package that happened since last release                                                                           |
-| Get Release Type | Based on the received data and updated dependencies,  calculates the release type according to semantic versioning                                       |
-| Prepare          | Performs any action that prepares all packages for publication after all versions are patched  (You can build packages here, omit devDeps, you name it). |
-| Publish          | Publishes individual package to destination                                                                                                              |
-| Post-Publish     | Performs any actions on successful publishing  (You can generate new tag, publish release notes, send web hooks and every other side effect here)        |
+| Step             | Description                                                                                                                                                            |
+|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Setup            | Sets up the plugin, checks all the conditions necessary for work.                                                                                                      |
+| Get Last Release | Obtains latest released version for each package (Usually from tags analysis)                                                                                          |
+| Extract commits  | Extracts commits relevant to specific package that happened since last release                                                                                         |
+| Get Release Type | Based on the received data and updated dependencies,  calculates the release type according to semantic versioning                                                     |
+| Prepare          | Performs any action that prepares all or individual packages for publication after all versions are patched  (You can build packages here, omit devDeps, you name it). |
+| Publish          | Publishes individual package to destination                                                                                                                            |
+| Post-Publish     | Performs any actions on successful publishing  (You can generate new tag, publish release notes, send web hooks and every other side effect here)                      |
 
 > All plugins must implement MonoPubPlugin interface, containing plugin name and implementation of one or more specified steps.
 > Detailed interface description can be found [here](https://github.com/SavelevMatthew/mono-pub/blob/main/packages/mono-pub/src/types/plugins.ts) 
@@ -114,10 +114,12 @@ const git = require('@mono-pub/git')
 const npm = require('@mono-pub/npm')
 const commitAnalyzer = require('@mono-pub/commit-analyzer')
 
+
+/** @type {import('mono-pub').MonoPubPlugin} */
 const builder = {
     name: '@mono-pub/local-builder',
-    async prepare(_, ctx) {
-        await execa('yarn', ['build'], { cwd: ctx.cwd })
+    async prepareSingle({ targetPackage }) {
+        await execa('yarn', ['build'], { cwd: targetPackage.location })
     },
 }
 
@@ -161,11 +163,16 @@ const git = require('@mono-pub/git')
 const github = require('@mono-pub/github')
 const npm = require('@mono-pub/npm')
 const commitAnalyzer = require('@mono-pub/commit-analyzer')
+const { getExecutionOrder } = require('mono-pub/utils')
 
+/** @type {import('mono-pub').MonoPubPlugin} */
 const builder = {
-    name: '@mono-pub/local-builder',
-    async prepare(_, ctx) {
-        await execa('yarn', ['build'], { cwd: ctx.cwd })
+    name: '@mono-pub/local-turborepo-builder',
+    async prepareAll({ foundPackages }, ctx) {
+        const batches = getExecutionOrder(foundPackages, { batching: true })
+        for (const batch of batches) {
+            await execa('yarn', ['build', ...batch.map(pkg => `--filter=${pkg.name}`)], { cwd: ctx.cwd })
+        }
     },
 }
 

packages/mono-pub/src/index.ts

@@ -3,8 +3,8 @@ import isEqual from 'lodash/isEqual'
 import { getAllPackages } from '@/utils/path'
 import getLogger from '@/logger'
 import { CombinedPlugin } from '@/utils/plugins'
-import { getDependencies, getReleaseOrder, patchPackageDeps } from '@/utils/deps'
-import { getNewVersion, versionToString } from '@/utils/versions'
+import { getDependencies, getExecutionOrder, patchPackageDeps } from '@/utils/deps'
+import { getNewVersion, versionToString, isPackageChanged } from '@/utils/versions'
 
 import type {
     MonoPubPlugin,
@@ -66,19 +66,19 @@ export default async function publish(
     )
     logger.log('Calculating release order based on packages dependencies and devDependencies...')
     let packagesWithDeps: Record<string, PackageInfoWithDependencies> = {}
-    let releaseOrder: Array<string> = []
+    let releaseOrder: Array<BasePackageInfo> = []
 
     try {
         packagesWithDeps = await getDependencies(packages)
-        releaseOrder = getReleaseOrder(packagesWithDeps)
+        releaseOrder = getExecutionOrder(Object.values(packagesWithDeps))
     } catch (err) {
         if (err instanceof Error) {
             logger.error(err.message)
         }
         throw err
     }
 
-    logger.success(`Packages release order: [${releaseOrder.map((pkg) => `"${pkg}"`).join(', ')}]`)
+    logger.success(`Packages release order: [${releaseOrder.map((pkg) => `"${pkg.name}"`).join(', ')}]`)
 
     logger.success(
         `Found ${plugins.length} plugins to form release chain: [${plugins
@@ -107,7 +107,7 @@ export default async function publish(
     const releaseTypes: Record<string, ReleaseType> = {}
     const newVersions: Record<string, LatestReleasedVersion> = {}
 
-    for (const pkgName of releaseOrder) {
+    for (const { name: pkgName } of releaseOrder) {
         const scopedLogger = scopedContexts[pkgName].logger
 
         const latestRelease = get(latestReleases, pkgName, null)
@@ -150,13 +150,28 @@ export default async function publish(
         await patchPackageDeps(pkg, newVersions, latestReleases)
     }
 
-    await releaseChain.prepare(packages, context)
+    const foundPackages = Object.values(packagesWithDeps)
+    const changedPackages = foundPackages.filter(({ name }) => {
+        const newVersion = newVersions[name]
+        const releaseType = releaseTypes[name]
+        const oldVersion = latestReleases[name]
+
+        return isPackageChanged(newVersion, oldVersion, releaseType)
+    })
+
+    await releaseChain.prepareAll(
+        {
+            foundPackages,
+            changedPackages,
+        },
+        context
+    )
 
-    for (const packageName of releaseOrder) {
+    for (const { name: packageName } of releaseOrder) {
         const newVersion = newVersions[packageName]
         const releaseType = releaseTypes[packageName]
         const oldVersion = latestReleases[packageName]
-        if (releaseType === 'none' || !newVersion || isEqual(newVersion, oldVersion)) {
+        if (!isPackageChanged(newVersion, oldVersion, releaseType)) {
             continue
         }
 

packages/mono-pub/src/types/plugins.ts

@@ -6,10 +6,22 @@ import type {
     PackageInfoWithLatestRelease,
     ReleaseType,
     ReleasedPackageInfo,
+    PackageInfoWithDependencies,
 } from './packages'
 
 type Awaitable<T> = T | Promise<T>
 
+export type PrepareAllInfo = {
+    /** All packages found by filter */
+    foundPackages: Array<PackageInfoWithDependencies>
+    /** Packages, which will actually be published */
+    changedPackages: Array<PackageInfoWithDependencies>
+}
+
+export type PrepareSingleInfo = PrepareAllInfo & {
+    targetPackage: BasePackageInfo
+}
+
 export interface MonoPubPlugin {
     /**
      * Name of plugin
@@ -51,12 +63,29 @@ export interface MonoPubPlugin {
 
     /**
      * Prepares packages for publishing. Usually includes build process.
+     * Most suitable for scenarios in monorepos with existing orchestrator, such as TurboRepo,
+     * where multiple packages can be built at once.
      * NOTE: This step is triggered once for all packages, not for each package individually
-     * @param packages {Array<BasePackageInfo>} List of packages containing its name and location (absolute path to package.json)
+     * NOTE: You can get execution order with or without batches by using getExecutionOrder util from mono-pub/utils
+     * @param info {PrepareAllInfo} Information about all packages found, as well as packages that will be published immediately.
+     * List of packages containing its name and location (absolute path to package.json)
+     * @param ctx {MonoPubContext} Execution context. Used to obtain cwd, env and logger
+     * @return {void}
+     */
+    prepareAll?(info: PrepareAllInfo, ctx: MonoPubContext): Awaitable<void>
+
+    /**
+     * Prepares individual package for publishing. Usually includes build process.
+     * Most suitable for scenarios in monorepos without, where you can just execute "yarn build" one by one.
+     * Order of execution is controlled for mono-pub, so you can ensure,
+     * that all package dependencies are built before package itself
+     * NOTE: This step is triggered once for all packages, not for each package individually
+     * @param info {PrepareSingleInfo} Information about all packages found, packages that will be directly published,
+     * and the current package being prepared
      * @param ctx {MonoPubContext} Execution context. Used to obtain cwd, env and logger
      * @return {void}
      */
-    prepare?(packages: Array<BasePackageInfo>, ctx: MonoPubContext): Awaitable<void>
+    prepareSingle?(info: PrepareSingleInfo, ctx: MonoPubContext): Awaitable<void>
 
     /**
      * Publishes package to a specific registry

packages/mono-pub/src/utils/deps.spec.ts

@@ -2,7 +2,7 @@ import fs from 'fs'
 import path from 'path'
 import { dirSync } from 'tmp'
 import { faker } from '@faker-js/faker'
-import { getDependencies, getReleaseOrder, patchPackageDeps } from './deps'
+import { getDependencies, getExecutionOrder, patchPackageDeps } from './deps'
 import { getNewVersion, versionToString } from './versions'
 
 import type { DirResult } from 'tmp'
@@ -28,6 +28,7 @@ describe('Dependencies utils', () => {
     let pkg1Info: BasePackageInfo
     let pkg2Info: BasePackageInfo
     let pkg3Info: BasePackageInfo
+    let pkg4Info: BasePackageInfo
     beforeEach(() => {
         tmpDir = dirSync({ unsafeCleanup: true })
         pkg1Info = {
@@ -43,7 +44,13 @@ describe('Dependencies utils', () => {
             location: path.join(tmpDir.name, 'packages/pkg3', 'package.json'),
         }
 
+        pkg4Info = {
+            name: '@scope/pkg4',
+            location: path.join(tmpDir.name, 'packages/pkg4', 'package.json'),
+        }
+
         writePackageJson({ name: pkg1Info.name, version: '0.0.0-development' }, 'packages/pkg1', tmpDir.name)
+        writePackageJson({ name: pkg4Info.name, version: '0.0.0-development' }, 'packages/pkg4', tmpDir.name)
         writePackageJson(
             {
                 name: pkg2Info.name,
@@ -78,13 +85,17 @@ describe('Dependencies utils', () => {
     })
     describe('getDependencies', () => {
         it('Should determine deps of packages correctly', async () => {
-            const deps = await getDependencies([pkg1Info, pkg2Info, pkg3Info])
+            const deps = await getDependencies([pkg1Info, pkg2Info, pkg3Info, pkg4Info])
             expect(deps).toEqual(
                 expect.objectContaining({
                     [pkg1Info.name]: {
                         ...pkg1Info,
                         dependsOn: [],
                     },
+                    [pkg4Info.name]: {
+                        ...pkg4Info,
+                        dependsOn: [],
+                    },
                     [pkg2Info.name]: {
                         ...pkg2Info,
                         dependsOn: expect.arrayContaining([{ name: pkg1Info.name, type: 'dep', value: 'workspace:^' }]),
@@ -99,6 +110,7 @@ describe('Dependencies utils', () => {
                 })
             )
             expect(deps[pkg1Info.name].dependsOn).toHaveLength(0)
+            expect(deps[pkg4Info.name].dependsOn).toHaveLength(0)
             expect(deps[pkg2Info.name].dependsOn).toHaveLength(1)
             expect(deps[pkg3Info.name].dependsOn).toHaveLength(2)
         })
@@ -118,20 +130,25 @@ describe('Dependencies utils', () => {
             )
         })
     })
-    describe('getReleaseOrder', () => {
+    describe('getExecutionOrder', () => {
         it('Should determine release order from leafs to root of deps tree', async () => {
             const deps = await getDependencies([pkg3Info, pkg2Info, pkg1Info])
-            const order = getReleaseOrder(deps)
-            expect(order).toEqual([pkg1Info.name, pkg2Info.name, pkg3Info.name])
+            const order = getExecutionOrder(Object.values(deps))
+            expect(order).toEqual([pkg1Info, pkg2Info, pkg3Info])
         })
         it('Should throw if cyclic deps found', async () => {
             const deps = await getDependencies([pkg3Info, pkg2Info, pkg1Info])
             deps[pkg1Info.name].dependsOn.push({ name: pkg3Info.name, value: '1.0.0', type: 'dep' })
 
             expect(() => {
-                getReleaseOrder(deps)
+                getExecutionOrder(Object.values(deps))
             }).toThrow('The release cannot be done because of cyclic dependencies')
         })
+        it('Can batch tasks, which can be executed together', async () => {
+            const deps = await getDependencies([pkg3Info, pkg2Info, pkg1Info, pkg4Info])
+            const batches = getExecutionOrder(Object.values(deps), { batching: true })
+            expect(batches).toEqual([[pkg1Info, pkg4Info], [pkg2Info], [pkg3Info]])
+        })
     })
     describe('patchPackageDeps', () => {
         it('Should patch package with new version or version from latest release', async () => {

packages/mono-pub/src/utils/deps.ts

@@ -34,39 +34,57 @@ export async function getDependencies(
     return result
 }
 
-export function getReleaseOrder(packages: Record<string, PackageInfoWithDependencies>): Array<string> {
-    const order: Array<string> = []
+type ExecutionOrder<T extends boolean | undefined> = T extends true
+    ? Array<Array<BasePackageInfo>>
+    : Array<BasePackageInfo>
 
-    const deps = new Map<string, Array<string>>()
-    for (const pkg of Object.values(packages)) {
-        deps.set(
+type TaskPlanningOptions<T extends boolean | undefined> = {
+    batching?: T
+}
+
+export function getExecutionOrder<T extends boolean | undefined = undefined>(
+    packages: Array<PackageInfoWithDependencies>,
+    options?: TaskPlanningOptions<T>
+): ExecutionOrder<T> {
+    const batches: Array<Array<BasePackageInfo>> = []
+    const pkgMap = Object.fromEntries(packages.map((pkg) => [pkg.name, pkg]))
+
+    const dependencies = new Map<string, Array<string>>()
+    for (const pkg of packages) {
+        dependencies.set(
             pkg.name,
             pkg.dependsOn.map((dep) => dep.name)
         )
     }
 
-    while (deps.size > 0) {
-        const toRelease: Array<string> = []
-        for (const [key, value] of deps) {
-            if (value.length === 0) {
-                toRelease.push(key)
-                deps.delete(key)
+    while (dependencies.size > 0) {
+        const batch: Array<BasePackageInfo> = []
+        for (const [pkgName, pkgDeps] of dependencies) {
+            if (pkgDeps.length === 0) {
+                batch.push({ name: pkgName, location: pkgMap[pkgName].location })
+                dependencies.delete(pkgName)
             }
         }
-        if (toRelease.length === 0) {
+
+        if (batch.length === 0) {
             throw new Error('The release cannot be done because of cyclic dependencies')
-        } else {
-            order.push(...toRelease)
-            for (const [key, value] of deps) {
-                deps.set(
-                    key,
-                    value.filter((pkg) => !toRelease.includes(pkg))
-                )
-            }
         }
+
+        batches.push(batch)
+        const includedPackages = batch.map((pkg) => pkg.name)
+        for (const [pkgName, pkgDeps] of dependencies) {
+            dependencies.set(
+                pkgName,
+                pkgDeps.filter((depName) => !includedPackages.includes(depName))
+            )
+        }
+    }
+
+    if (options?.batching) {
+        return batches as ExecutionOrder<T>
     }
 
-    return order
+    return batches.flat() as ExecutionOrder<T>
 }
 
 export async function patchPackageDeps(

packages/mono-pub/src/utils/index.ts

@@ -1 +1,2 @@
+export { getExecutionOrder } from './deps'
 export { versionToString } from './versions'