Add MediaProxy support to serve authenticated Matrix media

.eslintrc.js

@@ -184,5 +184,5 @@ module.exports = {
             }
         }
     ],
-    "ignorePatterns": [".eslintrc.js", "widget/**/*"],
+    "ignorePatterns": [".eslintrc.js", "widget/**/*", "src/generate-signing-key.js"],
 };

.github/workflows/lint.yml

@@ -13,6 +13,6 @@ jobs:
     - name: Use Node.js
       uses: actions/setup-node@v3
       with:
-        node-version: 16
+        node-version: 20
     - run: yarn --ignore-scripts --pure-lockfile --strict-semver
     - run: yarn lint

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
       matrix:
-        node_version: [16, 18]
+        node_version: [20]
    steps:
     - uses: actions/checkout@v2
     - name: Use Node.js
@@ -24,7 +24,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
       matrix:
-        node_version: [16, 18]
+        node_version: [20]
    steps:
     - uses: actions/checkout@v2
     - name: Use Node.js
@@ -38,7 +38,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
       matrix:
-        node_version: [16, 18]
+        node_version: [20]
    steps:
     - uses: actions/checkout@v2
     - name: Use Node.js
@@ -51,5 +51,5 @@ jobs:
         docker run --detach --publish 5432:5432 \
           --env POSTGRES_PASSWORD=pass \
           --env POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-          postgres:13
+          postgres:16
     - run: yarn test:postgres

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18-bullseye-slim AS BUILD
+FROM node:20-bullseye-slim AS BUILD
 
 # git is needed to install Half-Shot/slackdown
 RUN apt update && apt install -y git
@@ -13,7 +13,7 @@ COPY ./widget /build/widget/
 
 RUN yarn build
 
-FROM node:18-bullseye-slim
+FROM node:20-bullseye-slim
 
 VOLUME /data/ /config/
 

changelog.d/775.removal

@@ -0,0 +1 @@
+Drop support for Node 16 and 18, require Node 20.

config/config.sample.yaml

@@ -55,6 +55,17 @@ db:
   #key_file: /path/to/tls.key
   #crt_file: /path/to/tls.crt
 
+mediaProxy:
+  # To generate a .jwk file:
+  # $ node src/generate-signing-key.js > signingkey.jwk
+  signingKeyPath: "signingkey.jwk"
+  # How long should the generated URLs be valid for
+  ttlSeconds: 3600
+  # The port for the media proxy to listen on
+  bindPort: 11111
+  # The publically accessible URL to the media proxy
+  publicUrl: "https://slack.bridge/media"
+
 # Real Time Messaging API (RTM)
 # Optional if slack_hook_port and inbound_uri_prefix are defined, required otherwise.
 #

config/slack-config-schema.yaml

@@ -34,6 +34,18 @@ properties:
         type: number
       inactive_after_days:
         type: number
+  mediaProxy:
+    type: "object"
+    properties:
+      signingKeyPath:
+        type: "string"
+      ttlSeconds:
+        type: "integer"
+      bindPort:
+        type: "integer"
+      publicUrl:
+        type: "string"
+    required: ["signingKeyPath", "bindPort", "publicUrl"]
   rtm:
     type: object
     required: ["enable"]

package.json

@@ -3,7 +3,7 @@
   "version": "2.1.2",
   "description": "A Matrix <--> Slack bridge",
   "engines": {
-    "node": ">=16 <=18"
+    "node": ">=20"
   },
   "main": "app.js",
   "scripts": {
@@ -35,64 +35,65 @@
   "homepage": "https://github.com/matrix-org/matrix-appservice-slack#readme",
   "dependencies": {
     "@alloc/quick-lru": "^5.2.0",
-    "@slack/logger": "^3.0.0",
-    "@slack/rtm-api": "^6.0.0",
+    "@slack/logger": "^4.0.0",
+    "@slack/rtm-api": "^6.2.1",
     "@slack/web-api": "^6.7.2",
     "Slackdown": "git+https://[email protected]/half-shot/slackdown.git",
     "ajv": "^8.12.0",
-    "axios": "^0.27.2",
-    "classnames": "^2.3.2",
+    "axios": "^1.6.8",
+    "classnames": "^2.5.1",
     "escape-string-regexp": "^4.0.0",
-    "https-proxy-agent": "^5.0.1",
-    "matrix-appservice-bridge": "^8.1.2",
-    "matrix-widget-api": "^1.1.1",
-    "minimist": "^1.2.6",
+    "https-proxy-agent": "^7.0.4",
+    "matrix-appservice-bridge": "^10.3.1",
+    "matrix-bot-sdk": "^0.7.1",
+    "matrix-widget-api": "^1.6.0",
+    "minimist": "^1.2.8",
     "nedb": "^1.8.0",
-    "node-emoji": "^1.10.0",
+    "node-emoji": "^2.1.3",
     "nunjucks": "^3.2.4",
     "p-queue": "^6.0.0",
-    "pg-promise": "^10.11.1",
-    "randomstring": "^1.2.1",
+    "pg-promise": "^11.5.5",
+    "randomstring": "^1.3.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "url-join": "^5.0.0",
-    "uuid": "^8.3.2",
-    "yargs": "17.5.1"
+    "uuid": "^9.0.1",
+    "yargs": "17.7.2"
   },
   "devDependencies": {
-    "@tailwindcss/forms": "^0.5.3",
-    "@tsconfig/node16": "^1.0.3",
-    "@types/chai": "^4.2.21",
-    "@types/js-yaml": "^4.0.2",
-    "@types/mocha": "^9.1.1",
-    "@types/nedb": "^1.8.12",
-    "@types/node": "^18.6.1",
-    "@types/node-emoji": "^1.8.1",
-    "@types/nunjucks": "^3.1.5",
-    "@types/randomstring": "^1.1.7",
-    "@types/react": "^18.0.27",
-    "@types/react-dom": "^18.0.10",
-    "@types/uuid": "^8.3.1",
-    "@types/yargs": "17.0.10",
-    "@typescript-eslint/eslint-plugin": "^5.0.0",
-    "@typescript-eslint/parser": "^5.50.0",
-    "@vitejs/plugin-react": "^3.1.0",
-    "autoprefixer": "^10.4.13",
+    "@tailwindcss/forms": "^0.5.7",
+    "@tsconfig/node20": "^20.1.3",
+    "@types/chai": "^4.3.14",
+    "@types/js-yaml": "^4.0.9",
+    "@types/mocha": "^10.0.6",
+    "@types/nedb": "^1.8.16",
+    "@types/node": "^20.11.30",
+    "@types/node-emoji": "^2.1.0",
+    "@types/nunjucks": "^3.2.6",
+    "@types/randomstring": "^1.1.12",
+    "@types/react": "^18.2.71",
+    "@types/react-dom": "^18.2.22",
+    "@types/uuid": "^9.0.8",
+    "@types/yargs": "17.0.32",
+    "@typescript-eslint/eslint-plugin": "^7.4.0",
+    "@typescript-eslint/parser": "^7.4.0",
+    "@vitejs/plugin-react": "^4.2.1",
+    "autoprefixer": "^10.4.19",
     "chai": "^4.3.4",
-    "eslint": "^8.20.0",
-    "eslint-plugin-jsdoc": "^39.3.3",
+    "eslint": "^8.57.0",
+    "eslint-plugin-jsdoc": "^48.2.1",
     "eslint-plugin-prefer-arrow": "^1.2.3",
-    "eslint-plugin-react": "^7.32.2",
+    "eslint-plugin-react": "^7.34.1",
     "eslint-plugin-react-hooks": "^4.6.0",
     "js-yaml": "^4.1.0",
     "mocha": "^10.0.0",
+    "postcss": "^8.4.38",
+    "prom-client": "^15.1.0",
     "nyc": "^15.1.0",
-    "postcss": "^8.4.21",
-    "prom-client": "^14.0.1",
     "source-map-support": "^0.5.19",
-    "tailwindcss": "^3.2.4",
-    "ts-node": "^10.1.0",
-    "typescript": "^4.4.3",
-    "vite": "^4.1.1"
+    "tailwindcss": "^3.4.1",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.3",
+    "vite": "^5.2.6"
   }
 }

src/BridgedRoom.ts

@@ -18,7 +18,7 @@
 import { Logger, Intent } from "matrix-appservice-bridge";
 import { SlackGhost } from "./SlackGhost";
 import { Main, METRIC_SENT_MESSAGES } from "./Main";
-import { default as substitutions, getFallbackForMissingEmoji, IMatrixToSlackResult } from "./substitutions";
+import { default as substitutions, IMatrixToSlackResult } from "./substitutions";
 import * as emoji from "node-emoji";
 import { ISlackMessageEvent, ISlackEvent, ISlackFile } from "./BaseSlackHandler";
 import { WebAPIPlatformError, WebClient } from "@slack/web-api";
@@ -269,7 +269,7 @@
         return null;
     }
 
     public async onMatrixReaction(message: any): Promise<void> {
         const relatesTo = message.content["m.relates_to"];
         const eventStore = this.main.datastore;
         const event = await eventStore.getEventByMatrixId(message.room_id, relatesTo.event_id);
@@ -306,7 +306,7 @@
         // bot user. Search for #fix_reactions_as_bot.
         const res = await client.reactions.add({
             as_user: false,
-            channel: this.slackChannelId,
+            channel: this.slackChannelId!,
             name: emojiKeyName,
             timestamp: event.slackTs,
         });
@@ -329,7 +329,7 @@
         });
     }
 
     public async onMatrixRedaction(message: any): Promise<void> {
         const clientForRequest = await this.getClientForRequest(message.sender);
         if (!clientForRequest) {
             log.warn("No client to handle redaction");
@@ -383,7 +383,7 @@
         }
     }
 
     public async onMatrixEdit(message: any): Promise<boolean> {
         const clientForRequest = await this.getClientForRequest(message.sender);
         if (!clientForRequest) {
             log.warn("No client to handle edit");
@@ -438,7 +438,7 @@
         return true;
     }
 
     public async onMatrixMessage(message: any): Promise<boolean> {
         const puppetedClient = await this.main.clientFactory.getClientForUser(this.SlackTeamId!, message.sender);
         if (!this.slackWebhookUri && !this.botClient && !puppetedClient) { return false; }
         const slackClient = puppetedClient || this.botClient;
@@ -710,7 +710,7 @@
             if (ghostChanged) {
                 await this.main.fixDMMetadata(this, ghost);
             }
-            this.slackSendLock = this.slackSendLock.then(() => {
+            this.slackSendLock = this.slackSendLock.then(async () => {
                 // Check again
                 if (this.recentSlackMessages.includes(message.ts)) {
                     // We sent this, ignore
@@ -743,7 +743,7 @@
             return;
         }
 
-        let reactionKey = emoji.emojify(`:${message.reaction}:`, getFallbackForMissingEmoji);
+        let reactionKey = emoji.emojify(`:${message.reaction}:`);
         // Element uses the default thumbsup and thumbsdown reactions with an appended variant character.
         if (reactionKey === '👍' || reactionKey === '👎') {
             reactionKey += '\ufe0f'.normalize(); // VARIATION SELECTOR-16
@@ -833,7 +833,7 @@
     }
 
     private setValue<T>(key: string, value: T) {
         const sneakyThis = this as any;
         if (sneakyThis[key] === value) {
             return;
         }
@@ -1156,7 +1156,7 @@
         Strip out reply fallbacks. Borrowed from
         https://github.com/turt2live/matrix-js-bot-sdk/blob/master/src/preprocessors/RichRepliesPreprocessor.ts
     */
     private async stripMatrixReplyFallback(event: any): Promise<any> {
         if (!event.content?.body) {
             return event;
         }
@@ -1193,7 +1193,7 @@
         Given an event which is in reply to something else return the event ID of the
         top most event in the reply chain, i.e. the one without a relates to.
     */
     private async findParentReply(message: any, depth = 0): Promise<string> {
         const MAX_DEPTH = 10;
         // Extract the referenced event
         if (!message.content) { return message.event_id; }
@@ -1269,7 +1269,7 @@
         msgtype: "m.image",
         url,
         // TODO: Define some matrix types
     } as any;
 
     if (file.original_w) {
         message.info.w = file.original_w;

src/IConfig.ts

@@ -109,4 +109,11 @@ export interface IConfig {
         onboard_users?: boolean;
         direct_messages?: AllowDenyConfig;
     }
+
+    mediaProxy: {
+        signingKeyPath: string;
+        ttlSeconds?: number;
+        bindPort: number;
+        publicUrl: string;
+    }
 }

src/Main.ts

@@ -15,13 +15,16 @@ limitations under the License.
 */
 
 import {
-    Bridge, BridgeBlocker, PrometheusMetrics, StateLookup,
+    Bridge, BridgeBlocker, PrometheusMetrics, StateLookup, MediaProxy,
     Logger, Intent, UserMembership, WeakEvent, PresenceEvent,
     AppService, AppServiceRegistration, UserActivityState, UserActivityTracker,
-    UserActivityTrackerConfig, MembershipQueue, PowerLevelContent, StateLookupEvent } from "matrix-appservice-bridge";
+    UserActivityTrackerConfig, MembershipQueue, PowerLevelContent, StateLookupEvent,
+} from "matrix-appservice-bridge";
 import { Gauge, Counter } from "prom-client";
 import * as path from "path";
+import * as fs from "fs";
 import * as randomstring from "randomstring";
+import { webcrypto } from "node:crypto";
 import { WebClient } from "@slack/web-api";
 import { IConfig, CACHING_DEFAULTS } from "./IConfig";
 import { OAuth2 } from "./OAuth2";
@@ -148,6 +151,8 @@ export class Main {
     public slackRtm?: SlackRTMHandler;
     private slackHookHandler?: SlackHookHandler;
 
+    public mediaProxy?: MediaProxy;
+
     private provisioner: Provisioner;
 
     private bridgeBlocker?: BridgeBlocker;
@@ -333,6 +338,22 @@ export class Main {
         );
     }
 
+    private async initialiseMediaProxy(config: IConfig['mediaProxy']): Promise<void> {
+        const jwk = JSON.parse(fs.readFileSync(config.signingKeyPath, "utf8").toString());
+        const signingKey = await webcrypto.subtle.importKey('jwk', jwk, {
+            name: 'HMAC',
+            hash: 'SHA-512',
+        }, true, ['sign', 'verify']);
+        const publicUrl = new URL(config.publicUrl);
+
+        this.mediaProxy = new MediaProxy({
+            publicUrl,
+            signingKey,
+            ttl: config.ttlSeconds ? (config.ttlSeconds * 1000) : undefined,
+        }, this.bridge.getIntent().matrixClient);
+        await this.mediaProxy.start(config.bindPort);
+    }
+
     public teamIsUsingRtm(teamId: string): boolean {
         return (this.slackRtm !== undefined) && this.slackRtm.teamIsUsingRtm(teamId);
     }
@@ -1142,6 +1163,17 @@ export class Main {
             path: "/ready",
         });
 
+        if (this.config.mediaProxy) {
+            await this.initialiseMediaProxy(this.config.mediaProxy).catch(err => {
+                throw Error(`Failed to start Media Proxy: ${err}`);
+            });
+        } else {
+            log.warn(
+                "Media Proxy not configured: media bridging to Slack won't work on servers requiring authenticated media " +
+                "(default since Synapse v1.120.0)"
+            );
+        }
+
 
         await this.pingBridge();
 

src/SlackRTMHandler.ts

@@ -236,7 +236,7 @@ export class SlackRTMHandler extends SlackEventHandler {
                 const intent = await room.getIntentForRoom();
                 // We only want to act on trivial messages
                 // This can be asyncronous to the handling of the message.
-                intent.getStateEvent(room.MatrixRoomId, 'm.room.member', puppet.matrixId, true).then((state) => {
+                intent.getStateEvent(room.MatrixRoomId, 'm.room.member', puppet.matrixId, true).then(async (state) => {
                     if (!['invite', 'join'].includes(state?.membership)) {
                         // Automatically invite the user the room.
                         log.info(`User ${puppet.matrixId} is not in ${room.MatrixRoomId}/${room.SlackChannelId}, inviting`);

src/generate-signing-key.js

@@ -0,0 +1,11 @@
+const webcrypto = require('node:crypto');
+
+async function main() {
+    const key = await webcrypto.subtle.generateKey({
+        name: 'HMAC',
+        hash: 'SHA-512',
+    }, true, ['sign', 'verify']);
+    console.log(JSON.stringify(await webcrypto.subtle.exportKey('jwk', key), undefined, 4));
+}
+
+main().then(() => process.exit(0)).catch(err => { throw err });

src/substitutions.ts

@@ -25,14 +25,6 @@ const log = new Logger("substitutions");
 const ATTACHMENT_TYPES = ["m.audio", "m.video", "m.file", "m.image"];
 const PILL_REGEX = /<a href="https:\/\/matrix\.to\/#\/(#|@|\+)([^"]+)">([^<]+)<\/a>/g;
 
-/**
- * Will return the emoji's name within ':'.
- * @param name The emoji's name.
- */
-export const getFallbackForMissingEmoji = (name: string): string => (
-    `:${name}:`
-);
-
 interface PillItem {
     id: string;
     text: string;
@@ -74,7 +66,7 @@ class Substitutions {
             body = url ? body.replace(file.permalink, url) : body;
         }
 
-        body = emoji.emojify(body, getFallbackForMissingEmoji);
+        body = emoji.emojify(body);
 
         return body;
     }
@@ -181,7 +173,12 @@ class Substitutions {
             // in this case.
             return null;
         }
-        const url = main.getUrlForMxc(event.content.url, main.encryptRoom);
+        let url: string;
+        if (main.mediaProxy) {
+            url = await main.mediaProxy.generateMediaUrl(event.content.url).then(u => u.toString());
+        } else {
+            url = main.getUrlForMxc(event.content.url, main.encryptRoom);
+        }
         if (main.encryptRoom) {
             return {
                 encrypted_file: url,