fix(mysql): make sure migrations are run in order when run concurrently

Now we either lock all or none of the migrations to run,
to make sure they are not out of order when multiple instances of Emigrate run concurrently.

Author: joakimbeng

.changeset/bright-poems-approve.md

@@ -0,0 +1,5 @@
+---
+'@emigrate/mysql': patch
+---
+
+Make sure we can initialize multiple running instances of Emigrate using @emigrate/mysql concurrently without issues with creating the history table (for instance in a Kubernetes environment and/or with a Percona cluster).

.changeset/yellow-walls-tease.md

@@ -0,0 +1,5 @@
+---
+'@emigrate/mysql': patch
+---
+
+Either lock all or none of the migrations to run to make sure they run in order when multiple instances of Emigrate runs concurrently (for instance in a Kubernetes environment)

.github/workflows/ci.yaml

@@ -15,6 +15,18 @@ jobs:
       TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
       DO_NOT_TRACK: 1
 
+    services:
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: root
+          MYSQL_DATABASE: emigrate
+          MYSQL_USER: emigrate
+          MYSQL_PASSWORD: emigrate
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping -h localhost" --health-interval=10s --health-timeout=5s --health-retries=5
+
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -26,11 +38,22 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v4
         with:
-          node-version: 20.9.0
+          node-version: 22.15.0
           cache: 'pnpm'
 
       - name: Install dependencies
         run: pnpm install
 
+      - name: Wait for MySQL to be ready
+        run: |
+          for i in {1..30}; do
+            nc -z localhost 3306 && echo "MySQL is up!" && break
+            echo "Waiting for MySQL..."
+            sleep 2
+          done
+
       - name: Checks
+        env:
+          MYSQL_HOST: localhost
+          MYSQL_PORT: 3306
         run: pnpm checks

.github/workflows/release.yaml

@@ -30,7 +30,7 @@ jobs:
       - name: Setup Node.js environment
         uses: actions/setup-node@v4
         with:
-          node-version: 20.9.0
+          node-version: 22.15.0
           cache: 'pnpm'
 
       - name: Install Dependencies

package.json

@@ -37,7 +37,7 @@
   "bugs": "https://github.com/aboviq/emigrate/issues",
   "license": "MIT",
   "volta": {
-    "node": "20.9.0",
+    "node": "22.15.0",
     "pnpm": "9.4.0"
   },
   "packageManager": "pnpm@9.4.0",
@@ -80,6 +80,7 @@
     "lint-staged": "15.2.0",
     "npm-run-all": "4.1.5",
     "prettier": "3.1.1",
+    "testcontainers": "10.24.2",
     "tsx": "4.15.7",
     "turbo": "2.0.5",
     "typescript": "5.5.2",

packages/cli/src/commands/remove.test.ts

@@ -11,6 +11,7 @@ import {
   StorageInitError,
 } from '../errors.js';
 import {
+  assertErrorEqualEnough,
   getErrorCause,
   getMockedReporter,
   getMockedStorage,
@@ -199,6 +200,11 @@ function assertPreconditionsFailed(reporter: Mocked<Required<EmigrateReporter>>,
   assert.strictEqual(reporter.onMigrationSkip.mock.calls.length, 0, 'Total pending and skipped');
   assert.strictEqual(reporter.onFinished.mock.calls.length, 1, 'Finished called once');
   const [entries, error] = reporter.onFinished.mock.calls[0]?.arguments ?? [];
+  // hackety hack:
+  if (finishedError) {
+    finishedError.stack = error?.stack;
+  }
+
   assert.deepStrictEqual(error, finishedError, 'Finished error');
   const cause = getErrorCause(error);
   const expectedCause = finishedError?.cause;
@@ -288,14 +294,7 @@ function assertPreconditionsFulfilled(
   assert.strictEqual(reporter.onMigrationSkip.mock.calls.length, 0, 'Total pending and skipped');
   assert.strictEqual(reporter.onFinished.mock.calls.length, 1, 'Finished called once');
   const [entries, error] = reporter.onFinished.mock.calls[0]?.arguments ?? [];
-  assert.deepStrictEqual(error, finishedError, 'Finished error');
-  const cause = getErrorCause(error);
-  const expectedCause = finishedError?.cause;
-  assert.deepStrictEqual(
-    cause,
-    expectedCause ? deserializeError(expectedCause) : expectedCause,
-    'Finished error cause',
-  );
+  assertErrorEqualEnough(error, finishedError, 'Finished error');
   assert.strictEqual(entries?.length, expected.length, 'Finished entries length');
   assert.deepStrictEqual(
     entries.map((entry) => `${entry.name} (${entry.status})`),

packages/cli/src/commands/up.test.ts

@@ -1,13 +1,6 @@
 import { describe, it, mock } from 'node:test';
 import assert from 'node:assert';
-import {
-  type EmigrateReporter,
-  type Storage,
-  type Plugin,
-  type SerializedError,
-  type MigrationMetadataFinished,
-} from '@emigrate/types';
-import { deserializeError } from 'serialize-error';
+import { type EmigrateReporter, type Storage, type Plugin, type MigrationMetadataFinished } from '@emigrate/types';
 import { version } from '../get-package-info.js';
 import {
   BadOptionError,
@@ -16,15 +9,14 @@ import {
   MigrationHistoryError,
   MigrationRunError,
   StorageInitError,
-  toSerializedError,
 } from '../errors.js';
 import {
   type Mocked,
   toEntry,
   toMigrations,
   getMockedReporter,
   getMockedStorage,
-  getErrorCause,
+  assertErrorEqualEnough,
 } from '../test-utils.js';
 import upCommand from './up.js';
 
@@ -930,31 +922,21 @@ function assertPreconditionsFulfilled(
   for (const [index, entry] of failedEntries.entries()) {
     if (entry.status === 'failed') {
       const error = reporter.onMigrationError.mock.calls[index]?.arguments[1];
-      assert.deepStrictEqual(error, entry.error, 'Error');
-      const cause = entry.error?.cause;
-      assert.deepStrictEqual(error?.cause, cause ? deserializeError(cause) : cause, 'Error cause');
+      assertErrorEqualEnough(error, entry.error, 'Error');
 
       if (entry.started) {
         const [finishedMigration, error] = storage.onError.mock.calls[index]?.arguments ?? [];
         assert.strictEqual(finishedMigration?.name, entry.name);
         assert.strictEqual(finishedMigration?.status, entry.status);
-        assertErrorEqualEnough(error, entry.error);
+        assertErrorEqualEnough(error, entry.error, `Entry error (${entry.name})`);
       }
     }
   }
 
   assert.strictEqual(reporter.onMigrationSkip.mock.calls.length, pending + skipped, 'Total pending and skipped');
   assert.strictEqual(reporter.onFinished.mock.calls.length, 1, 'Finished called once');
   const [entries, error] = reporter.onFinished.mock.calls[0]?.arguments ?? [];
-  assertErrorEqualEnough(error, finishedError);
-
-  const cause = getErrorCause(error);
-  const expectedCause = finishedError?.cause;
-  assert.deepStrictEqual(
-    cause,
-    expectedCause ? deserializeError(expectedCause) : expectedCause,
-    'Finished error cause',
-  );
+  assertErrorEqualEnough(error, finishedError, 'Finished error');
   assert.strictEqual(entries?.length, expected.length, 'Finished entries length');
   assert.deepStrictEqual(
     entries.map((entry) => `${entry.name} (${entry.status})`),
@@ -995,33 +977,6 @@ function assertPreconditionsFailed(
   assert.strictEqual(reporter.onMigrationSkip.mock.calls.length, 0, 'Total pending and skipped');
   assert.strictEqual(reporter.onFinished.mock.calls.length, 1, 'Finished called once');
   const [entries, error] = reporter.onFinished.mock.calls[0]?.arguments ?? [];
-  assert.deepStrictEqual(error, finishedError, 'Finished error');
-  const cause = getErrorCause(error);
-  const expectedCause = finishedError?.cause;
-  assert.deepStrictEqual(
-    cause,
-    expectedCause ? deserializeError(expectedCause) : expectedCause,
-    'Finished error cause',
-  );
+  assertErrorEqualEnough(error, finishedError, 'Finished error');
   assert.strictEqual(entries?.length, 0, 'Finished entries length');
 }
-
-function assertErrorEqualEnough(actual?: Error | SerializedError, expected?: Error) {
-  if (expected === undefined) {
-    assert.strictEqual(actual, undefined);
-    return;
-  }
-
-  const {
-    cause: actualCause,
-    stack: actualStack,
-    ...actualError
-  } = actual instanceof Error ? toSerializedError(actual) : actual ?? {};
-  const { cause: expectedCause, stack: expectedStack, ...expectedError } = toSerializedError(expected);
-  // @ts-expect-error Ignore
-  const { stack: actualCauseStack, ...actualCauseRest } = actualCause ?? {};
-  // @ts-expect-error Ignore
-  const { stack: expectedCauseStack, ...expectedCauseRest } = expectedCause ?? {};
-  assert.deepStrictEqual(actualError, expectedError);
-  assert.deepStrictEqual(actualCauseRest, expectedCauseRest);
-}

packages/cli/src/test-utils.ts

@@ -1,5 +1,6 @@
 import { mock, type Mock } from 'node:test';
 import path from 'node:path';
+import assert from 'node:assert';
 import {
   type SerializedError,
   type EmigrateReporter,
@@ -9,6 +10,7 @@ import {
   type NonFailedMigrationHistoryEntry,
   type Storage,
 } from '@emigrate/types';
+import { toSerializedError } from './errors.js';
 
 export type Mocked<T> = {
   // @ts-expect-error - This is a mock
@@ -110,3 +112,23 @@ export function toEntries(
 ): MigrationHistoryEntry[] {
   return names.map((name) => (typeof name === 'string' ? toEntry(name, status) : name));
 }
+
+export function assertErrorEqualEnough(actual?: Error | SerializedError, expected?: Error, message?: string): void {
+  if (expected === undefined) {
+    assert.strictEqual(actual, undefined);
+    return;
+  }
+
+  const {
+    cause: actualCause,
+    stack: actualStack,
+    ...actualError
+  } = actual instanceof Error ? toSerializedError(actual) : actual ?? {};
+  const { cause: expectedCause, stack: expectedStack, ...expectedError } = toSerializedError(expected);
+  // @ts-expect-error Ignore
+  const { stack: actualCauseStack, ...actualCauseRest } = actualCause ?? {};
+  // @ts-expect-error Ignore
+  const { stack: expectedCauseStack, ...expectedCauseRest } = expectedCause ?? {};
+  assert.deepStrictEqual(actualError, expectedError, message);
+  assert.deepStrictEqual(actualCauseRest, expectedCauseRest, message ? `${message} (cause)` : undefined);
+}

packages/mysql/package.json

@@ -17,12 +17,16 @@
   },
   "files": [
     "dist",
-    "!dist/*.tsbuildinfo"
+    "!dist/*.tsbuildinfo",
+    "!dist/**/*.test.js",
+    "!dist/tests/*"
   ],
   "scripts": {
     "build": "tsc --pretty",
     "build:watch": "tsc --pretty --watch",
-    "lint": "xo --cwd=../.. $(pwd)"
+    "lint": "xo --cwd=../.. $(pwd)",
+    "test-disabled": "glob -c \"node --import tsx --test-reporter spec --test\" \"./src/**/*.test.ts\"",
+    "test:watch": "glob -c \"node --watch --import tsx --test-reporter spec --test\" \"./src/**/*.test.ts\""
   },
   "keywords": [
     "emigrate",

packages/mysql/src/index.test.ts

@@ -0,0 +1,92 @@
+import assert from 'node:assert';
+import path from 'node:path';
+import { before, after, describe, it } from 'node:test';
+import type { MigrationMetadata } from '@emigrate/types';
+import { startDatabase, stopDatabase } from './tests/database.js';
+import { createMysqlStorage } from './index.js';
+
+let db: { port: number; host: string };
+
+describe('emigrate-mysql', async () => {
+  before(
+    async () => {
+      db = await startDatabase();
+    },
+    { timeout: 60_000 },
+  );
+
+  after(
+    async () => {
+      await stopDatabase();
+    },
+    { timeout: 10_000 },
+  );
+
+  describe('migration locks', async () => {
+    it('either locks none or all of the given migrations', async () => {
+      const { initializeStorage } = createMysqlStorage({
+        table: 'migrations',
+        connection: {
+          host: db.host,
+          user: 'emigrate',
+          password: 'emigrate',
+          database: 'emigrate',
+          port: db.port,
+        },
+      });
+
+      const [storage1, storage2] = await Promise.all([initializeStorage(), initializeStorage()]);
+
+      const migrations = toMigrations('/emigrate', 'migrations', [
+        '2023-10-01-01-test.js',
+        '2023-10-01-02-test.js',
+        '2023-10-01-03-test.js',
+        '2023-10-01-04-test.js',
+        '2023-10-01-05-test.js',
+        '2023-10-01-06-test.js',
+        '2023-10-01-07-test.js',
+        '2023-10-01-08-test.js',
+        '2023-10-01-09-test.js',
+        '2023-10-01-10-test.js',
+        '2023-10-01-11-test.js',
+        '2023-10-01-12-test.js',
+        '2023-10-01-13-test.js',
+        '2023-10-01-14-test.js',
+        '2023-10-01-15-test.js',
+        '2023-10-01-16-test.js',
+        '2023-10-01-17-test.js',
+        '2023-10-01-18-test.js',
+        '2023-10-01-19-test.js',
+        '2023-10-01-20-test.js',
+      ]);
+
+      const [locked1, locked2] = await Promise.all([storage1.lock(migrations), storage2.lock(migrations)]);
+
+      assert.strictEqual(
+        locked1.length === 0 || locked2.length === 0,
+        true,
+        'One of the processes should have no locks',
+      );
+      assert.strictEqual(
+        locked1.length === 20 || locked2.length === 20,
+        true,
+        'One of the processes should have all locks',
+      );
+    });
+  });
+});
+
+function toMigration(cwd: string, directory: string, name: string): MigrationMetadata {
+  return {
+    name,
+    filePath: `${cwd}/${directory}/${name}`,
+    relativeFilePath: `${directory}/${name}`,
+    extension: path.extname(name),
+    directory,
+    cwd,
+  };
+}
+
+function toMigrations(cwd: string, directory: string, names: string[]): MigrationMetadata[] {
+  return names.map((name) => toMigration(cwd, directory, name));
+}