Skip to content

Refactor the configure_workers_and_start.py script used internally by Complement. #16803

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 20 commits into from
Closed

Refactor the configure_workers_and_start.py script used internally by Complement. #16803

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
b39a50a
Remove obsolete `"app"` from worker templates
reivilibre Nov 16, 2023
a22eb7d
Convert worker templates into dataclass
reivilibre Nov 16, 2023
ba3b6a4
Use a lambda for the worker name rather than search and replace later
reivilibre Nov 16, 2023
67d4fc8
Collapse WORKERS_CONFIG by removing entries with defaults
reivilibre Nov 16, 2023
94a85b3
Convert listener_resources and endpoint_patterns to Set[str]
reivilibre Nov 16, 2023
26073fa
Tweak comments
reivilibre Nov 16, 2023
8b74639
Add `merge_into`
reivilibre Nov 16, 2023
f38297b
Remove special logic for adding stream_writers: just make it part of …
reivilibre Nov 16, 2023
7d8824e
Rename function to add_worker_to_instance_map given reduction of scope
reivilibre Nov 16, 2023
f49dbc7
Add `sharding_allowed` to the WorkerTemplate rather than having a sep…
reivilibre Nov 16, 2023
3bb21a9
Use `merge_into` when adding workers to the shared config
reivilibre Nov 16, 2023
fbafde8
Promote mark_filepath to constant
reivilibre Nov 16, 2023
321d359
Add a --generate-only option
reivilibre Nov 16, 2023
259a808
Docstring on WorkerTemplate
reivilibre Dec 6, 2023
3a46cf0
Fix comment and mutation bug on merge_worker_template_configs
reivilibre Dec 6, 2023
2f1d727
Update comment on `merged`
reivilibre Dec 6, 2023
ad4bb0e
Tweak `instantiate_worker_template`, both in name, description and va…
reivilibre Dec 6, 2023
2ff1de3
Newsfile
reivilibre Jan 10, 2024
29541fd
Move `stream_writers` to their own field in the WorkerTemplate
reivilibre Jan 17, 2024
c91ab4b
Remove `merge_into` and just have `merged` which copies inputs to avo…
reivilibre Jan 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docker/conf-workers/synapse.supervisord.conf.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ command=/usr/local/bin/python -m synapse.app.complement_fork_starter
--config-path="{{ main_config_path }}"
--config-path=/conf/workers/shared.yaml
{%- for worker in workers %}
-- {{ worker.app }}
-- synapse.app.generic_worker
--config-path="{{ main_config_path }}"
--config-path=/conf/workers/shared.yaml
--config-path=/conf/workers/{{ worker.name }}.yaml
Expand Down Expand Up @@ -36,7 +36,7 @@ exitcodes=0

{% for worker in workers %}
[program:synapse_{{ worker.name }}]
command=/usr/local/bin/prefix-log /usr/local/bin/python -m {{ worker.app }}
command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.generic_worker
--config-path="{{ main_config_path }}"
--config-path=/conf/workers/shared.yaml
--config-path=/conf/workers/{{ worker.name }}.yaml
Expand Down
2 changes: 1 addition & 1 deletion docker/conf-workers/worker.yaml.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# Values will be change depending on whichever workers are selected when
# running that image.

worker_app: "{{ app }}"
worker_app: "synapse.app.generic_worker"
worker_name: "{{ name }}"

worker_listeners:
Expand Down
18 changes: 0 additions & 18 deletions docker/configure_workers_and_start.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,12 @@
# have to attach by instance_map to the master process and have client endpoints.
WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
"pusher": {
"app": "synapse.app.generic_worker",
"listener_resources": [],
"endpoint_patterns": [],
"shared_extra_conf": {},
"worker_extra_conf": "",
},
"user_dir": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client"],
"endpoint_patterns": [
"^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$"
Expand All @@ -108,7 +106,6 @@
"worker_extra_conf": "",
},
"media_repository": {
"app": "synapse.app.generic_worker",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, the documentation still thinks that you should use synapse.app.media_repository to configure this: https://element-hq.github.io/synapse/latest/workers.html#synapseappmedia_repository

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hrm, that's a headscratcher.

Complement does test the media endpoints and the tests pass.......?

But looking at

synapse/synapse/config/repository.py

Line 129 in f95cfd8

and config.get("worker_app") != "synapse.app.media_repository"
(the only place where synapse.app.media_repository seems to matter), you'd expect this to break things...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⇒ #16826

"listener_resources": ["media"],
"endpoint_patterns": [
"^/_matrix/media/",
Expand All @@ -126,7 +123,6 @@
"worker_extra_conf": "enable_media_repo: true",
},
"appservice": {
"app": "synapse.app.generic_worker",
"listener_resources": [],
"endpoint_patterns": [],
"shared_extra_conf": {
Expand All @@ -135,14 +131,12 @@
"worker_extra_conf": "",
},
"federation_sender": {
"app": "synapse.app.generic_worker",
"listener_resources": [],
"endpoint_patterns": [],
"shared_extra_conf": {},
"worker_extra_conf": "",
},
"synchrotron": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client"],
"endpoint_patterns": [
"^/_matrix/client/(v2_alpha|r0|v3)/sync$",
Expand All @@ -154,7 +148,6 @@
"worker_extra_conf": "",
},
"client_reader": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client"],
"endpoint_patterns": [
"^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$",
Expand Down Expand Up @@ -189,7 +182,6 @@
"worker_extra_conf": "",
},
"federation_reader": {
"app": "synapse.app.generic_worker",
"listener_resources": ["federation"],
"endpoint_patterns": [
"^/_matrix/federation/(v1|v2)/event/",
Expand All @@ -216,21 +208,18 @@
"worker_extra_conf": "",
},
"federation_inbound": {
"app": "synapse.app.generic_worker",
"listener_resources": ["federation"],
"endpoint_patterns": ["/_matrix/federation/(v1|v2)/send/"],
"shared_extra_conf": {},
"worker_extra_conf": "",
},
"event_persister": {
"app": "synapse.app.generic_worker",
"listener_resources": ["replication"],
"endpoint_patterns": [],
"shared_extra_conf": {},
"worker_extra_conf": "",
},
"background_worker": {
"app": "synapse.app.generic_worker",
"listener_resources": [],
"endpoint_patterns": [],
# This worker cannot be sharded. Therefore, there should only ever be one
Expand All @@ -239,7 +228,6 @@
"worker_extra_conf": "",
},
"event_creator": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client"],
"endpoint_patterns": [
"^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact",
Expand All @@ -253,14 +241,12 @@
"worker_extra_conf": "",
},
"frontend_proxy": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client", "replication"],
"endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"],
"shared_extra_conf": {},
"worker_extra_conf": "",
},
"account_data": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client", "replication"],
"endpoint_patterns": [
"^/_matrix/client/(r0|v3|unstable)/.*/tags",
Expand All @@ -270,14 +256,12 @@
"worker_extra_conf": "",
},
"presence": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client", "replication"],
"endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/presence/"],
"shared_extra_conf": {},
"worker_extra_conf": "",
},
"receipts": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client", "replication"],
"endpoint_patterns": [
"^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt",
Expand All @@ -287,14 +271,12 @@
"worker_extra_conf": "",
},
"to_device": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client", "replication"],
"endpoint_patterns": ["^/_matrix/client/(r0|v3|unstable)/sendToDevice/"],
"shared_extra_conf": {},
"worker_extra_conf": "",
},
"typing": {
"app": "synapse.app.generic_worker",
"listener_resources": ["client", "replication"],
"endpoint_patterns": [
"^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing"
Expand Down