Merge branch 'main' into hughns/syn2mas-captcha

Author: sandhose

tools/syn2mas/.nvmrc

@@ -1 +1 @@
-18
+22

tools/syn2mas/Dockerfile

@@ -1,6 +1,6 @@
 
 # Build Node.js app
-FROM --platform=${BUILDPLATFORM} docker.io/library/node:18-bookworm AS builder
+FROM --platform=${BUILDPLATFORM} docker.io/library/node:22-bookworm AS builder
 
 WORKDIR /syn2mas
 
@@ -10,7 +10,7 @@ RUN --network=default \
   npm ci
 
 # Install the production dependencies for each architecture we support
-FROM --platform=${BUILDPLATFORM} docker.io/library/node:18-bookworm AS deps
+FROM --platform=${BUILDPLATFORM} docker.io/library/node:22-bookworm AS deps
 
 WORKDIR /deps/arm64
 

tools/syn2mas/package-lock.json

@@ -32,10 +32,10 @@
     "start": "node dist/index.js"
   },
   "devDependencies": {
-    "@tsconfig/node18": "^18.2.2",
+    "@tsconfig/node22": "^22.0.0",
     "@tsconfig/strictest": "^2.0.2",
     "@types/command-line-args": "^5.2.2",
-    "@types/node": "^18.18.7",
+    "@types/node": "^22.0.0",
     "@typescript-eslint/eslint-plugin": "^7.3.1",
     "@typescript-eslint/parser": "^7.3.1",
     "eslint": "^8.52.0",
@@ -57,6 +57,7 @@
     "knex": "^3.0.1",
     "log4js": "^6.9.1",
     "pg": "^8.11.3",
+    "pg-query-stream": "^4.6.0",
     "sqlite3": "^5.1.6",
     "ts-command-line-args": "^2.5.1",
     "yaml": "^2.3.3",

tools/syn2mas/package.json

@@ -96,17 +96,17 @@ export async function advisor(): Promise<void> {
   );
   if (guestUsers > 0) {
     error(
-      `Synapse database contains ${guestUsers} guest users which aren't supported by MAS: https://github.com/matrix-org/matrix-authentication-service/issues/1445`,
+      `Synapse database contains ${guestUsers} guest users which aren't supported by MAS: https://github.com/element-hq/matrix-authentication-service/issues/1445`,
     );
   }
   if (synapseConfig.allow_guest_access) {
     if (guestUsers > 0) {
       error(
-        "Synapse config allows guest access which isn't supported by MAS: https://github.com/matrix-org/matrix-authentication-service/issues/1445",
+        "Synapse config allows guest access which isn't supported by MAS: https://github.com/element-hq/matrix-authentication-service/issues/1445",
       );
     } else {
       error(
-        "Synapse config allows guest access which isn't supported by MAS, but no guest users were found in the database so the option could be disabled: https://github.com/matrix-org/matrix-authentication-service/issues/1445",
+        "Synapse config allows guest access which isn't supported by MAS, but no guest users were found in the database so the option could be disabled: https://github.com/element-hq/matrix-authentication-service/issues/1445",
       );
     }
   }
@@ -140,7 +140,7 @@ export async function advisor(): Promise<void> {
   );
   if (usersWithoutEmailAddress > 0) {
     warn(
-      `Synapse database contains ${usersWithoutEmailAddress} users without a verified email address who will need to verify their email address before they can login after migration: https://github.com/matrix-org/matrix-authentication-service/issues/1505`,
+      `Synapse database contains ${usersWithoutEmailAddress} users without a verified email address who will need to verify their email address before they can login after migration: https://github.com/element-hq/matrix-authentication-service/issues/1505`,
     );
   }
 

tools/syn2mas/src/advisor.mts

@@ -192,13 +192,7 @@ export async function migrate(): Promise<void> {
     );
   }
 
-  // Get all Synapse users, except appservice owned users who don't need to be migrated
-  const synapseUsers = await synapse
-    .select("*")
-    .from<SUser>("users")
-    .whereNull("appservice_id");
-  log.info(`Found ${synapseUsers.length} users in Synapse`);
-  for (const user of synapseUsers) {
+  async function migrateUser(user: SUser): Promise<void> {
     const localpart = user.name.split(":")[0].substring(1);
     log.info(`Processing user ${user.name} as ${localpart}`);
 
@@ -430,10 +424,43 @@ export async function migrate(): Promise<void> {
       }
     }
   }
+
+  // this is a workaround to get the list of columns that we care about from the SUser type
+  const SUserColumns: Record<keyof SUser, undefined> = {
+    name: undefined,
+    password_hash: undefined,
+    admin: undefined,
+    is_guest: undefined,
+    deactivated: undefined,
+    creation_ts: undefined,
+    appservice_id: undefined,
+  };
+
+  // Get all Synapse users, except appservice owned users who don't need to be migrated
+  const synapseUserQuery = synapse
+    .select(Object.keys(SUserColumns) as (keyof SUser)[])
+    .from<SUser>("users")
+    .whereNull("appservice_id");
+
+  let synapseUsers = 0;
+  if (synapseConfig.database.name === "sqlite3") {
+    // SQLite doesn't support streaming
+    const synapseUserRows = (await synapseUserQuery) as unknown as SUser[];
+    for (const user of synapseUserRows) {
+      synapseUsers += 1;
+      await migrateUser(user);
+    }
+  } else {
+    // Stream users from the database
+    const synapseUserStream = synapseUserQuery.stream();
+    for await (const user of synapseUserStream) {
+      synapseUsers += 1;
+      await migrateUser(user as unknown as SUser);
+    }
+  }
+
   log.info(
-    `Completed migration ${args.dryRun ? "dry-run " : ""}of ${
-      synapseUsers.length
-    } users with ${fatals} fatals and ${warnings.length} warnings:`,
+    `Completed migration ${args.dryRun ? "dry-run " : ""}of ${synapseUsers} users with ${fatals} fatals and ${warnings.length} warnings:`,
   );
   warnings.forEach((w) => log.warn(w));
   if (fatals > 0) {

tools/syn2mas/src/migrate.mts

@@ -1,7 +1,7 @@
 {
   "extends": [
     "@tsconfig/strictest/tsconfig.json",
-    "@tsconfig/node18/tsconfig.json",
+    "@tsconfig/node22/tsconfig.json",
   ],
   "compilerOptions": {
     "noEmit": true,

tools/syn2mas/tsconfig.eslint.json

@@ -1,7 +1,7 @@
 {
   "extends": [
     "@tsconfig/strictest/tsconfig.json",
-    "@tsconfig/node18/tsconfig.json"
+    "@tsconfig/node22/tsconfig.json"
   ],
   "compilerOptions": {
     "outDir": "dist",