Skip to content
This repository was archived by the owner on Apr 26, 2024. It is now read-only.

Commit d6b9378

Browse files
reivilibrereivilibre
committed
Convert listener_resources and endpoint_patterns to Set[str]
1 parent fd97df3 commit d6b9378

File tree

1 file changed

+44
-46
lines changed

1 file changed

+44
-46
lines changed

‎docker/configure_workers_and_start.py‎

Lines changed: 44 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,8 @@
9090
# have to attach by instance_map to the master process and have client endpoints.
9191
@dataclass
9292
class WorkerTemplate:
93-
listener_resources: List[str] = field(default_factory=list)
94-
endpoint_patterns: List[str] = field(default_factory=list)
93+
listener_resources: Set[str] = field(default_factory=set)
94+
endpoint_patterns: Set[str] = field(default_factory=set)
9595
# (worker_name) -> {}
9696
shared_extra_conf: Callable[[str], Dict[str, Any]] = lambda _worker_name: {}
9797
worker_extra_conf: str = ""
@@ -100,24 +100,24 @@ class WorkerTemplate:
100100
WORKERS_CONFIG: Dict[str, WorkerTemplate] = {
101101
"pusher": WorkerTemplate(),
102102
"user_dir": WorkerTemplate(
103-
listener_resources=["client"],
104-
endpoint_patterns=[
103+
listener_resources={"client"},
104+
endpoint_patterns={
105105
"^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$"
106-
],
106+
},
107107
shared_extra_conf=lambda worker_name: {
108108
"update_user_directory_from_worker": worker_name
109109
},
110110
),
111111
"media_repository": WorkerTemplate(
112-
listener_resources=["media"],
113-
endpoint_patterns=[
112+
listener_resources={"media"},
113+
endpoint_patterns={
114114
"^/_matrix/media/",
115115
"^/_synapse/admin/v1/purge_media_cache$",
116116
"^/_synapse/admin/v1/room/.*/media.*$",
117117
"^/_synapse/admin/v1/user/.*/media.*$",
118118
"^/_synapse/admin/v1/media/.*$",
119119
"^/_synapse/admin/v1/quarantine_media/.*$",
120-
],
120+
},
121121
# The first configured media worker will run the media background jobs
122122
shared_extra_conf=lambda worker_name: {
123123
"enable_media_repo": False,
@@ -132,17 +132,17 @@ class WorkerTemplate:
132132
),
133133
"federation_sender": WorkerTemplate(),
134134
"synchrotron": WorkerTemplate(
135-
listener_resources=["client"],
136-
endpoint_patterns=[
135+
listener_resources={"client"},
136+
endpoint_patterns={
137137
"^/_matrix/client/(v2_alpha|r0|v3)/sync$",
138138
"^/_matrix/client/(api/v1|v2_alpha|r0|v3)/events$",
139139
"^/_matrix/client/(api/v1|r0|v3)/initialSync$",
140140
"^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$",
141-
],
141+
},
142142
),
143143
"client_reader": WorkerTemplate(
144-
listener_resources=["client"],
145-
endpoint_patterns=[
144+
listener_resources={"client"},
145+
endpoint_patterns={
146146
"^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$",
147147
"^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/joined_members$",
148148
"^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$",
@@ -170,11 +170,11 @@ class WorkerTemplate:
170170
"^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$",
171171
"^/_matrix/client/(r0|v3|unstable)/capabilities$",
172172
"^/_matrix/client/(r0|v3|unstable)/notifications$",
173-
],
173+
},
174174
),
175175
"federation_reader": WorkerTemplate(
176-
listener_resources=["federation"],
177-
endpoint_patterns=[
176+
listener_resources={"federation"},
177+
endpoint_patterns={
178178
"^/_matrix/federation/(v1|v2)/event/",
179179
"^/_matrix/federation/(v1|v2)/state/",
180180
"^/_matrix/federation/(v1|v2)/state_ids/",
@@ -194,60 +194,60 @@ class WorkerTemplate:
194194
"^/_matrix/federation/(v1|v2)/user/devices/",
195195
"^/_matrix/federation/(v1|v2)/get_groups_publicised$",
196196
"^/_matrix/key/v2/query",
197-
],
197+
},
198198
),
199199
"federation_inbound": WorkerTemplate(
200-
listener_resources=["federation"],
201-
endpoint_patterns=["/_matrix/federation/(v1|v2)/send/"],
200+
listener_resources={"federation"},
201+
endpoint_patterns={"/_matrix/federation/(v1|v2)/send/"},
202202
),
203203
"event_persister": WorkerTemplate(
204-
listener_resources=["replication"],
204+
listener_resources={"replication"},
205205
),
206206
"background_worker": WorkerTemplate(
207207
# This worker cannot be sharded. Therefore, there should only ever be one
208208
# background worker. This is enforced for the safety of your database.
209209
shared_extra_conf=lambda worker_name: {"run_background_tasks_on": worker_name},
210210
),
211211
"event_creator": WorkerTemplate(
212-
listener_resources=["client"],
213-
endpoint_patterns=[
212+
listener_resources={"client"},
213+
endpoint_patterns={
214214
"^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact",
215215
"^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send",
216216
"^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
217217
"^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
218218
"^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
219219
"^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
220-
],
220+
},
221221
),
222222
"frontend_proxy": WorkerTemplate(
223-
listener_resources=["client", "replication"],
224-
endpoint_patterns=["^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"],
223+
listener_resources={"client", "replication"},
224+
endpoint_patterns={"^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"},
225225
),
226226
"account_data": WorkerTemplate(
227-
listener_resources=["client", "replication"],
228-
endpoint_patterns=[
227+
listener_resources={"client", "replication"},
228+
endpoint_patterns={
229229
"^/_matrix/client/(r0|v3|unstable)/.*/tags",
230230
"^/_matrix/client/(r0|v3|unstable)/.*/account_data",
231-
],
231+
},
232232
),
233233
"presence": WorkerTemplate(
234-
listener_resources=["client", "replication"],
235-
endpoint_patterns=["^/_matrix/client/(api/v1|r0|v3|unstable)/presence/"],
234+
listener_resources={"client", "replication"},
235+
endpoint_patterns={"^/_matrix/client/(api/v1|r0|v3|unstable)/presence/"},
236236
),
237237
"receipts": WorkerTemplate(
238-
listener_resources=["client", "replication"],
239-
endpoint_patterns=[
238+
listener_resources={"client", "replication"},
239+
endpoint_patterns={
240240
"^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt",
241241
"^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers",
242-
],
242+
},
243243
),
244244
"to_device": WorkerTemplate(
245-
listener_resources=["client", "replication"],
246-
endpoint_patterns=["^/_matrix/client/(r0|v3|unstable)/sendToDevice/"],
245+
listener_resources={"client", "replication"},
246+
endpoint_patterns={"^/_matrix/client/(r0|v3|unstable)/sendToDevice/"},
247247
),
248248
"typing": WorkerTemplate(
249-
listener_resources=["client", "replication"],
250-
endpoint_patterns=["^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing"],
249+
listener_resources={"client", "replication"},
250+
endpoint_patterns={"^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing"},
251251
),
252252
}
253253

@@ -406,15 +406,11 @@ def merge_worker_template_configs(
406406
# copy existing_template without any replacements
407407
new_template: WorkerTemplate = dataclasses.replace(existing_template)
408408

409-
# merge the two lists, remove duplicates
410-
new_template.listener_resources = list(
411-
set(new_template.listener_resources + to_be_merged_template.listener_resources)
412-
)
409+
# add listener resources from the other set
410+
new_template.listener_resources |= to_be_merged_template.listener_resources
413411

414-
# merge the two lists, remove duplicates
415-
new_template.endpoint_patterns = list(
416-
set(new_template.endpoint_patterns + to_be_merged_template.endpoint_patterns)
417-
)
412+
# add endpoint patterns from the other set
413+
new_template.endpoint_patterns |= to_be_merged_template.endpoint_patterns
418414

419415
# merge dictionaries; the worker name will be replaced later
420416
new_template.shared_extra_conf = lambda worker_name: {
@@ -444,6 +440,8 @@ def insert_worker_name_for_worker_config(
444440
"""
445441
dict_to_edit = dataclasses.asdict(existing_template)
446442
dict_to_edit["shared_extra_conf"] = existing_template.shared_extra_conf(worker_name)
443+
dict_to_edit["endpoint_patterns"] = sorted(existing_template.endpoint_patterns)
444+
dict_to_edit["listener_resources"] = sorted(existing_template.listener_resources)
447445
return dict_to_edit
448446

449447

@@ -760,7 +758,7 @@ def generate_worker_files(
760758
# Map locations to upstreams (corresponding to worker types) in Nginx
761759
# but only if we use the appropriate worker type
762760
for worker_type in all_worker_types_in_use:
763-
for endpoint_pattern in WORKERS_CONFIG[worker_type].endpoint_patterns:
761+
for endpoint_pattern in sorted(WORKERS_CONFIG[worker_type].endpoint_patterns):
764762
nginx_locations[endpoint_pattern] = f"http://{worker_type}"
765763

766764
# For each worker type specified by the user, create config values and write it's

0 commit comments

Comments
 (0)