chore(MSC3911): Complement (#139)

Lot of things going on here:
1. Enable the new `media_repo_instances` list for media workers
2. Add the new test package directory in Complement
3. Turn on `msc3911_enabled`
4. Make sure to route to workers when that mode is enabled

There is an additional temporary commit to be removed before merging. It
exists to override the branch name of Complement to pull for testing.
Once this is merged it should automatically allow the Synapse branch of
`msc3911` to target `msc3911` branch on the Complement repo. Other
branches/PRs should pull from the `main` branch, unless there exists a
branch of exactly the same name on both repos. E.g. if my PR has the
branch name `jason/cool-fixes` it will search for that exact same branch
name on Complement before falling back to `main`

Author: jason-famedly

.ci/scripts/checkout_complement.sh

@@ -21,5 +21,5 @@ for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/head
     continue
   fi
 
-  (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
+  (wget -O - "https://github.com/famedly/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
 done

.github/workflows/famedly-tests.yml

@@ -25,7 +25,7 @@ jobs:
       - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
       - uses: matrix-org/setup-python-poetry@v2
         with:
-          python-version: "3.x"
+          python-version: "3.13"
           poetry-version: "2.1.1"
           extras: "all"
       - run: poetry run scripts-dev/generate_sample_config.sh --check
@@ -49,7 +49,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.x"
+          python-version: "3.13"
       - run: .ci/scripts/check_lockfile.py
 
   lint:
@@ -63,6 +63,7 @@ jobs:
         uses: matrix-org/setup-python-poetry@v2
         with:
           poetry-version: "2.1.1"
+          python-version: "3.13"
           install-project: "false"
 
       - name: Run ruff check
@@ -91,6 +92,7 @@ jobs:
           # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
           # To make CI green, err towards caution and install the project.
           install-project: "true"
+          python-version: "3.13"
           poetry-version: "2.1.1"
 
       # Cribbed from
@@ -124,6 +126,7 @@ jobs:
       - uses: matrix-org/setup-python-poetry@v2
         with:
           poetry-version: "2.1.1"
+          python-version: "3.13"
           extras: "all"
       - run: poetry run scripts-dev/check_pydantic_models.py
 
@@ -161,7 +164,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.x"
+          python-version: "3.13"
       - run: "pip install rstcheck"
       - run: "rstcheck --report-level=WARNING README.rst"
 

docker/complement/conf/workers-shared-extra.yaml.j2

@@ -129,6 +129,8 @@ experimental_features:
   msc3984_appservice_key_query: true
   # Invite filtering
   msc4155_enabled: true
+  # Media Attachment
+  msc3911_enabled: true
 
 server_notices:
   system_mxid_localpart: _server

docker/configure_workers_and_start.py

@@ -117,15 +117,15 @@
     },
     "media_repository": {
         "app": "synapse.app.generic_worker",
-        "listener_resources": ["media", "client"],
+        "listener_resources": ["media", "client", "replication"],
         "endpoint_patterns": [
             "^/_matrix/media/",
             "^/_synapse/admin/v1/purge_media_cache$",
             "^/_synapse/admin/v1/room/.*/media.*$",
             "^/_synapse/admin/v1/user/.*/media.*$",
             "^/_synapse/admin/v1/media/.*$",
             "^/_synapse/admin/v1/quarantine_media/.*$",
-            "^/_matrix/client/v1/media/.*$",
+            "^/_matrix/client/(v1|unstable/.*)/media/.*$",
             "^/_matrix/federation/v1/media/.*$",
         ],
         # The first configured media worker will run the media background jobs
@@ -448,6 +448,9 @@ def add_worker_roles_to_shared_config(
     if "federation_sender" in worker_types_set:
         shared_config.setdefault("federation_sender_instances", []).append(worker_name)
 
+    if "media_repository" in worker_types_set:
+        shared_config.setdefault("media_repo_instances", []).append(worker_name)
+
     # Update the list of stream writers. It's convenient that the name of the worker
     # type is the same as the stream to write. Iterate over the whole list in case there
     # is more than one.
@@ -468,6 +471,17 @@ def add_worker_roles_to_shared_config(
                     "host": "localhost",
                     "port": worker_port,
                 }
+        if worker == "media_repository":
+            # Just like for stream_writers, media workers now need to be on the instance_map
+            if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
+                instance_map[worker_name] = {
+                    "path": f"/run/worker.{worker_port}",
+                }
+            else:
+                instance_map[worker_name] = {
+                    "host": "localhost",
+                    "port": worker_port,
+                }
 
 
 def merge_worker_template_configs(

scripts-dev/complement.sh

@@ -230,6 +230,7 @@ test_packages=(
     ./tests/msc3967
     ./tests/msc4140
     ./tests/msc4155
+    ./tests/msc3911
 )
 
 # Enable dirty runs, so tests will reuse the same container where possible.