Merge pull request #4 from ECADInfra/hardening-and-workers

Harden Dockerfile and integrate worker orchestration

Author: jevonearth

.github/workflows/docker-publish.yml

@@ -30,7 +30,6 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Log in to Container Registry
-        if: startsWith(github.ref, 'refs/tags/v')
         uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
@@ -44,16 +43,17 @@ jobs:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           annotations: |
             org.opencontainers.image.description=Synapse homeserver with Ed25519 Beacon auth for Tezos dApp/wallet relay
-          # Tag scheme: push v1.147.1-ecad.1 -> ghcr tags: v1.147.1-ecad.1, latest
           tags: |
             type=semver,pattern={{version}}
             type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+            type=ref,event=pr
+            type=ref,event=branch
 
       - name: Build and push Docker image
         uses: docker/build-push-action@v6
         with:
           context: .
-          push: ${{ startsWith(github.ref, 'refs/tags/v') }}
+          push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           annotations: ${{ steps.meta.outputs.annotations }}

Dockerfile

@@ -4,15 +4,37 @@ LABEL org.opencontainers.image.description="Synapse homeserver with Ed25519 Beac
 LABEL org.opencontainers.image.source="https://github.com/ECADInfra/beacon-synapse"
 LABEL org.opencontainers.image.licenses="AGPL-3.0-only"
 
-# Install dependencies for crypto auth provider
-RUN apt-get update && apt-get install -y libsodium-dev gcc netcat-openbsd gettext-base && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Install Python packages
-RUN pip install --no-cache-dir psycopg2 pysodium
+# netcat-openbsd: TCP readiness checks (wait-for.sh)
+# gettext-base: envsubst for config templating
+# nginx-light, redis-server, supervisor: worker mode orchestration
+# No gcc/libsodium-dev needed: PyNaCl (with bundled libsodium) and psycopg2
+# are already installed in the base Synapse image.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    netcat-openbsd gettext-base \
+    nginx-light redis-server supervisor \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Configure nginx for worker mode (remove default site, log to stdout)
+RUN rm -f /etc/nginx/sites-enabled/default && \
+    ln -sf /dev/stdout /var/log/nginx/access.log && \
+    ln -sf /dev/stderr /var/log/nginx/error.log
+
+# Symlink binaries to expected locations (configure_workers_and_start.py expects these)
+RUN mkdir -p /etc/supervisor/conf.d && \
+    ln -s /usr/bin/supervisord /usr/local/bin/supervisord && \
+    ln -s /usr/bin/supervisorctl /usr/local/bin/supervisorctl && \
+    ln -s /usr/bin/redis-server /usr/local/bin/redis-server
 
 # Create keys and data directories
 RUN mkdir -p /keys /data
 
+# Increase max event size (1MB instead of default 64KB).
+# Beacon messages can exceed the default Matrix PDU size limit.
+# Runs before COPY so code changes don't invalidate this layer.
+RUN sed -i 's/^MAX_PDU_SIZE = 65536$/MAX_PDU_SIZE = 1048576/' /usr/local/lib/python3.13/site-packages/synapse/api/constants.py && \
+    grep -q '^MAX_PDU_SIZE = 1048576$' /usr/local/lib/python3.13/site-packages/synapse/api/constants.py || \
+    (echo "FATAL: PDU size patch failed - 'MAX_PDU_SIZE = 65536' not found in constants.py. Upstream may have changed." >&2 && exit 1)
+
 # Copy custom modules (using Python 3.13 path for Element HQ image)
 COPY crypto_auth_provider.py /usr/local/lib/python3.13/site-packages/
 COPY beacon_info_module.py /usr/local/lib/python3.13/site-packages/
@@ -21,24 +43,20 @@ COPY beacon_monitor_module.py /usr/local/lib/python3.13/site-packages/
 # Copy configuration templates (envsubst at runtime) and static configs
 COPY homeserver.yaml /config/homeserver.yaml.template
 COPY synapse.log.config /config/
-COPY shared_config.yaml /config/shared_config.yaml.template
 
-# Copy worker configuration templates
-COPY workers /config/workers.template
-
-# Increase max event size (1MB instead of default 64KB).
-# Beacon messages can exceed the default Matrix PDU size limit.
-RUN sed -i 's/65536/1048576/' /usr/local/lib/python3.13/site-packages/synapse/api/constants.py && \
-    grep -q '1048576' /usr/local/lib/python3.13/site-packages/synapse/api/constants.py || \
-    (echo "FATAL: PDU size patch failed - 65536 not found in constants.py. Upstream may have changed." >&2 && exit 1)
+# Copy worker orchestration configs (Jinja2 templates for configure_workers_and_start.py)
+COPY conf-workers /conf/
+COPY configure_workers_and_start.py /usr/local/bin/
+COPY prefix-log /usr/local/bin/
 
 COPY wait-for.sh /usr/local/bin/
 COPY synctl_entrypoint.sh /usr/local/bin/
 
 # Expose ports:
-#   8008: HTTP (client and federation)
+#   8008: HTTP (client + federation; direct in single mode, nginx in worker mode)
+#   8080: Main process HTTP (worker mode only, internal)
+#   9469: Prometheus service discovery (worker mode, nginx)
 #   19090: Metrics for main process (when SYNAPSE_ENABLE_METRICS=1)
-#   19091-19094: Metrics for workers 1-4 (when SYNAPSE_ENABLE_METRICS=1 and SYNAPSE_WORKERS=true)
-EXPOSE 8008 19090 19091 19092 19093 19094
+EXPOSE 8008 8080 9469 19090
 
 ENTRYPOINT ["/usr/local/bin/synctl_entrypoint.sh"]

README.md

@@ -2,16 +2,6 @@
 
 Production-ready [Matrix](https://matrix.org/) homeserver image for operating [Beacon](https://tzip.tezosagora.org/proposal/tzip-10/) relay nodes. Beacon is the open communication standard connecting Tezos wallets and dApps, ratified as [TZIP-10](https://tzip.tezosagora.org/proposal/tzip-10/). Matrix provides the federated transport layer.
 
-This image is what [ECAD Infra](https://ecadinfra.com) runs in production. We publish it so that anyone can operate Beacon relay infrastructure using the same tooling we do.
-
-## Why operator diversity matters
-
-Beacon relay nodes are the infrastructure that every Tezos wallet-to-dApp connection depends on. When a user pairs a wallet with a dApp, that connection flows through Matrix relay servers.
-
-The Beacon network is stronger when relay infrastructure is operated by **multiple independent organizations** across different regions and hosting providers. Federation is a core design property of the Matrix protocol, and Beacon inherits it: wallets and dApps can communicate regardless of which operator's relay node they connect through. No single organization needs to be a bottleneck or a single point of failure.
-
-This image exists to make running a Beacon relay node straightforward. If you operate infrastructure in the Tezos ecosystem, consider running one.
-
 ## Provenance
 
 Beacon relay infrastructure was originally created by [Papers (AirGap)](https://papers.ch/) as part of the [Beacon SDK](https://github.com/airgap-it/beacon-sdk) ecosystem. Papers designed the protocol, built the SDK, and operated the first relay nodes, establishing the standard that the entire Tezos wallet and dApp ecosystem relies on today.
@@ -21,20 +11,51 @@ This image builds on [AirGap's beacon-node](https://github.com/airgap-it/beacon-
 ### What changed from upstream
 
 - **Synapse v1.98.0 to v1.147.1**: Upgraded to current release
+- **`crypto_auth_provider.py` v0.3**: PyNaCl-based Ed25519 auth (no gcc/libsodium-dev build deps), structured logfmt logging, race condition handling
 - **`beacon_monitor_module.py`**: Observability module for diagnosing connection and federation issues. Logs operational metadata (room lifecycle, membership changes, payload sizes, login events) in logfmt format. All Beacon message payloads are encrypted end-to-end between wallet and dApp using NaCl cryptobox before reaching the relay server; message content is not and cannot be logged. User and room identifiers are opaque hashes with no link to real-world identity.
 - **`beacon_info_module.py`**: HTTP endpoint exposing server region and known relay servers
-- **Worker mode**: Support for 4 generic workers behind the main process
+- **Worker mode**: Official Element HQ worker orchestration (supervisord + nginx + redis) for horizontal scaling
 - **`MAX_PDU_SIZE` patch**: 64KB to 1MB (Beacon messages can exceed the default Matrix limit)
 - **logfmt logging**: Structured log output for ingestion into Loki/Grafana/etc.
+- **SSRF protection**: `federation_ip_range_blacklist` covering RFC 1918, link-local, and loopback
 - **Robust entrypoint**: Template variable substitution, database readiness check, single-process or multi-worker modes
 
 ## Quick start
 
 ```bash
+# Single-process mode (default)
 docker compose -f docker-compose.example.yml up --build
+
+# Worker mode
+SYNAPSE_WORKERS=true docker compose -f docker-compose.example.yml up --build
+```
+
+This starts Synapse with PostgreSQL. The server will be available at `http://localhost:8008`.
+
+## Architecture
+
+### Single-process mode (default)
+
+```
+Client/Federation --> :8008 [Synapse] --> PostgreSQL
+```
+
+One Synapse process handles everything. Good for development and low-traffic deployments.
+
+### Worker mode
+
+```
+Client/Federation --> :8008 [nginx] --> :8080 [Synapse main]
+                                   \-> :18009+ [workers]
+                                        - synchrotron (sync)
+                                        - event_persister (writes)
+                                        - federation_inbound (federation)
+                     :9469 [nginx] --> Prometheus service discovery
+                     [redis] <------> inter-process replication
+                     [supervisord] --> manages all processes
 ```
 
-This starts Synapse with PostgreSQL and Redis. The server will be available at `http://localhost:8008`.
+Supervisord orchestrates the main Synapse process, worker processes, nginx, and redis, all inside a single container. nginx routes requests to the appropriate worker based on URL patterns. Redis handles inter-process replication.
 
 ## Configuration
 
@@ -52,11 +73,32 @@ The image uses template variables in `homeserver.yaml` that are substituted at s
 | `SERVER_REGION` | No | Region label for the `/beacon/info` endpoint |
 | `SYNAPSE_ENABLE_METRICS` | No | Set to `1` to expose Prometheus metrics on port 19090 |
 | `SYNAPSE_WORKERS` | No | Set to `true` to enable multi-worker mode |
+| `SYNAPSE_WORKER_TYPES` | No | Comma-separated worker types (default: `synchrotron:2,event_persister:1,federation_inbound:1`) |
 | `PUBLIC_BASEURL` | No | Public URL for federation (default: `https://SERVER_NAME`) |
 | `SERVE_WELLKNOWN` | No | Set to `true` to serve `.well-known/matrix/server` for Cloudflare |
 | `DB_CP_MIN` | No | Minimum database connections (default: `20`) |
 | `DB_CP_MAX` | No | Maximum database connections (default: `80`) |
 
+### Worker types
+
+The `SYNAPSE_WORKER_TYPES` variable accepts the official Element HQ worker type syntax:
+
+```bash
+# Simple list
+SYNAPSE_WORKER_TYPES="synchrotron,event_persister,federation_inbound"
+
+# With multipliers
+SYNAPSE_WORKER_TYPES="synchrotron:2,event_persister:2,federation_inbound:1"
+
+# Combined workers (merge types into one process)
+SYNAPSE_WORKER_TYPES="stream_writers=account_data+presence+typing"
+
+# Custom names
+SYNAPSE_WORKER_TYPES="sync=synchrotron:2,persist=event_persister:1"
+```
+
+Available types: `synchrotron`, `event_persister`, `federation_inbound`, `federation_sender`, `federation_reader`, `client_reader`, `event_creator`, `media_repository`, `user_dir`, `pusher`, `appservice`, `background_worker`, `account_data`, `presence`, `receipts`, `to_device`, `typing`, `push_rules`, `device_lists`, `thread_subscriptions`.
+
 ### Entrypoint options
 
 ```bash
@@ -72,11 +114,30 @@ docker run ghcr.io/ecadinfra/beacon-synapse --skip-templating
 
 ### Ports
 
-| Port | Service |
-|---|---|
-| 8008 | HTTP (client + federation) |
-| 19090 | Prometheus metrics (main process, when enabled) |
-| 19091-19094 | Prometheus metrics (workers 1-4, when enabled) |
+| Port | Mode | Service |
+|---|---|---|
+| 8008 | Both | HTTP (client + federation). Direct in single mode, nginx in worker mode |
+| 8080 | Worker | Main Synapse process (internal, behind nginx) |
+| 9469 | Worker | Prometheus service discovery + metrics proxy |
+| 19090 | Both | Prometheus metrics (main process, when enabled) |
+
+### Prometheus service discovery (worker mode)
+
+When `SYNAPSE_ENABLE_METRICS=1` and worker mode is active, port 9469 serves:
+
+- `GET /metrics/service_discovery` - JSON for Prometheus `http_sd_config`
+- `GET /metrics/worker/<name>` - Proxied metrics for each worker
+- `GET /metrics/worker/main` - Proxied metrics for the main process
+
+Prometheus config:
+
+```yaml
+scrape_configs:
+  - job_name: beacon-synapse
+    http_sd_configs:
+      - url: http://synapse:9469/metrics/service_discovery
+    honor_labels: true
+```
 
 ### Federation behind Cloudflare
 
@@ -93,9 +154,9 @@ This configures Synapse to:
 2. Set `public_baseurl` for proper federation discovery
 
 Other Matrix servers will then connect on port 443 instead of 8448. Ensure your reverse proxy routes:
-- `/.well-known/matrix/server` → Synapse (port 8008)
-- `/_matrix/federation/*` → Synapse (port 8008)
-- `/_matrix/client/*` → Synapse (port 8008)
+- `/.well-known/matrix/server` -> Synapse (port 8008)
+- `/_matrix/federation/*` -> Synapse (port 8008)
+- `/_matrix/client/*` -> Synapse (port 8008)
 
 **Important**: Configure Cloudflare to not challenge `/_matrix/federation/*` paths (these are server-to-server requests, not browsers).
 
@@ -142,7 +203,7 @@ Exposes `/_synapse/client/beacon/info` returning:
 If you want to operate a Beacon relay node for the Tezos ecosystem:
 
 1. Deploy this image with the configuration above
-2. Ensure port 8443 (or your federation port) is reachable from other relay nodes
+2. Ensure port 8448 (or 443 if using `.well-known` delegation) is reachable from other relay nodes
 3. Configure federation with existing operators so wallets and dApps on your node can communicate with the broader network
 4. Open an issue or reach out to coordinate federation peering
 

conf-workers/healthcheck.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/sh
+# This healthcheck script is designed to return OK when every 
+# host involved returns OK
+{%- for healthcheck_url in healthcheck_urls %}
+curl -fSs {{ healthcheck_url }} || exit 1
+{%- endfor %}

conf-workers/log.config

@@ -0,0 +1,62 @@
+version: 1
+formatters:
+    logfmt:
+{% if include_worker_name_in_log_line %}
+        format: 'ts=%(asctime)s worker={{ worker_name }} logger=%(name)s level=%(levelname)s request=%(request)s %(message)s'
+{% else %}
+        format: 'ts=%(asctime)s logger=%(name)s level=%(levelname)s request=%(request)s %(message)s'
+{% endif %}
+        datefmt: '%Y-%m-%dT%H:%M:%S'
+filters:
+    context:
+        (): synapse.logging.context.LoggingContextFilter
+        request: ""
+handlers:
+    console:
+        class: logging.StreamHandler
+        formatter: logfmt
+        filters: [context]
+{% if LOG_FILE_PATH %}
+    file:
+        class: logging.handlers.TimedRotatingFileHandler
+        formatter: logfmt
+        filters: [context]
+        filename: {{ LOG_FILE_PATH }}
+        when: midnight
+        backupCount: 3
+        encoding: utf8
+{% endif %}
+loggers:
+    synapse.storage.SQL:
+{% if SYNAPSE_LOG_SENSITIVE %}
+        level: {{ SYNAPSE_LOG_LEVEL or "DEBUG" }}
+{% else %}
+        level: ERROR
+{% endif %}
+
+    # Beacon modules: auth, monitoring, info
+    crypto_auth_provider:
+        level: INFO
+    beacon_monitor_module:
+        level: INFO
+    beacon_info_module:
+        level: INFO
+
+    # Synapse auth/login (failed logins, rate limiting)
+    synapse.handlers.auth:
+        level: WARNING
+    synapse.rest.client.login:
+        level: INFO
+
+{% if SYNAPSE_LOG_TESTING %}
+    synapse.handlers.typing:
+        level: DEBUG
+{% endif %}
+root:
+    level: {{ SYNAPSE_LOG_LEVEL or "ERROR" }}
+    handlers:
+        - console
+{% if LOG_FILE_PATH %}
+        - file
+{% endif %}
+disable_existing_loggers: false