diff --git a/poetry.lock b/poetry.lock index 63ef8573a075..7ab6d245fcb6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1367,7 +1367,7 @@ python-versions = ">=3.6" [[package]] name = "types-bleach" -version = "4.1.4" +version = "5.0.3" description = "Typing stubs for bleach" category = "dev" optional = false @@ -1993,10 +1993,7 @@ keyring = [ {file = "keyring-23.5.0.tar.gz", hash = "sha256:9012508e141a80bd1c0b6778d5c610dd9f8c464d75ac6774248500503f972fb9"}, ] ldap3 = [ - {file = "ldap3-2.9.1-py2.6.egg", hash = "sha256:5ab7febc00689181375de40c396dcad4f2659cd260fc5e94c508b6d77c17e9d5"}, - {file = "ldap3-2.9.1-py2.7.egg", hash = "sha256:2bc966556fc4d4fa9f445a1c31dc484ee81d44a51ab0e2d0fd05b62cac75daa6"}, {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, - {file = "ldap3-2.9.1-py3.9.egg", hash = "sha256:5630d1383e09ba94839e253e013f1aa1a2cf7a547628ba1265cb7b9a844b5687"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, ] lxml = [ @@ -2287,34 +2284,12 @@ psycopg2cffi-compat = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] pyasn1 = [ - {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, - {file = "pyasn1-0.4.8-py2.5.egg", hash = "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf"}, - {file = "pyasn1-0.4.8-py2.6.egg", hash = "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00"}, - {file = "pyasn1-0.4.8-py2.7.egg", hash = "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8"}, {file = "pyasn1-0.4.8-py2.py3-none-any.whl", hash = "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d"}, - {file = "pyasn1-0.4.8-py3.1.egg", hash = "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86"}, - {file = "pyasn1-0.4.8-py3.2.egg", hash = "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7"}, - {file = "pyasn1-0.4.8-py3.3.egg", hash = "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576"}, - {file = "pyasn1-0.4.8-py3.4.egg", hash = "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12"}, - {file = "pyasn1-0.4.8-py3.5.egg", hash = "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2"}, - {file = "pyasn1-0.4.8-py3.6.egg", hash = "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359"}, - {file = "pyasn1-0.4.8-py3.7.egg", hash = "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776"}, {file = "pyasn1-0.4.8.tar.gz", hash = "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba"}, ] pyasn1-modules = [ {file = "pyasn1-modules-0.2.8.tar.gz", hash = "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e"}, - {file = "pyasn1_modules-0.2.8-py2.4.egg", hash = "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199"}, - {file = "pyasn1_modules-0.2.8-py2.5.egg", hash = "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405"}, - {file = "pyasn1_modules-0.2.8-py2.6.egg", hash = "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb"}, - {file = "pyasn1_modules-0.2.8-py2.7.egg", hash = "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8"}, {file = "pyasn1_modules-0.2.8-py2.py3-none-any.whl", hash = "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74"}, - {file = "pyasn1_modules-0.2.8-py3.1.egg", hash = "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d"}, - {file = "pyasn1_modules-0.2.8-py3.2.egg", hash = "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45"}, - {file = "pyasn1_modules-0.2.8-py3.3.egg", hash = "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4"}, - {file = "pyasn1_modules-0.2.8-py3.4.egg", hash = "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811"}, - {file = "pyasn1_modules-0.2.8-py3.5.egg", hash = "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed"}, - {file = "pyasn1_modules-0.2.8-py3.6.egg", hash = "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0"}, - {file = "pyasn1_modules-0.2.8-py3.7.egg", hash = "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd"}, ] pycodestyle = [ {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, @@ -2452,6 +2427,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -2720,8 +2702,8 @@ typed-ast = [ {file = "typed_ast-1.5.2.tar.gz", hash = "sha256:525a2d4088e70a9f75b08b3f87a51acc9cde640e19cc523c7e41aa355564ae27"}, ] types-bleach = [ - {file = "types-bleach-4.1.4.tar.gz", hash = "sha256:2d30c2c4fb6854088ac636471352c9a51bf6c089289800d2a8060820a01cd43a"}, - {file = "types_bleach-4.1.4-py3-none-any.whl", hash = "sha256:edffe173ed6d7b6f3543036a96204a9319c3bf6c3645917b14274e43f000cc9b"}, + {file = "types-bleach-5.0.3.tar.gz", hash = "sha256:f7b3df8278efe176d9670d0f063a66c866c77577f71f54b9c7a320e31b1a7bbd"}, + {file = "types_bleach-5.0.3-py3-none-any.whl", hash = "sha256:5931525d03571f36b2bb40210c34b662c4d26c8fd6f2b1e1e83fe4d2d2fd63c7"}, ] types-commonmark = [ {file = "types-commonmark-0.9.2.tar.gz", hash = "sha256:b894b67750c52fd5abc9a40a9ceb9da4652a391d75c1b480bba9cef90f19fc86"}, diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 0963fb3bb4fb..cc54115e683c 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import time import urllib.parse from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Tuple @@ -308,6 +309,10 @@ async def push_bulk( "left": list(device_list_summary.left), } + if len(serialized_events) == 0 and len(ephemeral) == 0: + logger.info("Returning early on transaction: no events to send") + return True + try: await self.put_json( uri=uri, @@ -348,7 +353,19 @@ async def push_bulk( def _serialize( self, service: "ApplicationService", events: Iterable[EventBase] ) -> List[JsonDict]: + new_events = [] time_now = self.clock.time_msec() + + for event in events: + if int(round(time.time() * 1000)) - event.origin_server_ts > (15 * 60 * 1000): + logger.warning("Dropping event (due to age) %s" % event.event_id) + continue + if service.id != "github" and service.is_interested_in_user(event.sender) and event.sender.endswith(":t2bot.io"): + logger.warning("Dropping event (due to echo) %s" % event.event_id) + continue + logger.info("Allowing @ fallback: %s" % event.event_id) + new_events.append(event) + return [ serialize_event( e, @@ -367,5 +384,5 @@ def _serialize( ), ), ) - for e in events + for e in new_events ] diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 4dca711cd28d..9ee21de8437e 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -355,7 +355,7 @@ async def get_pdu( destinations: Iterable[str], event_id: str, room_version: RoomVersion, - timeout: Optional[int] = None, + timeout: Optional[int] = 15000, ) -> Optional[EventBase]: """Requests the PDU with given origin and ID from the remote home servers. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 907940e19eb0..075c9dd85148 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -85,7 +85,7 @@ # when processing incoming transactions, we try to handle multiple rooms in # parallel, up to this limit. -TRANSACTION_CONCURRENCY_LIMIT = 10 +TRANSACTION_CONCURRENCY_LIMIT = 50 # T2B: Raise from 10 logger = logging.getLogger(__name__) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index a6cb3ba58f0b..6f91a785418f 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -695,6 +695,11 @@ def send_presence_to_destinations( for destination in destinations: if destination == self.server_name: continue + + # T2B: Skip sending presence to servers we know don't support it + if destination == "matrix.org": + continue + if not self._federation_shard_config.should_handle( self._instance_name, destination ): diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 32074b8ca690..4893f5ccf7d3 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -105,7 +105,7 @@ async def get_room_state( ) async def get_event( - self, destination: str, event_id: str, timeout: Optional[int] = None + self, destination: str, event_id: str, timeout: Optional[int] = 15000 ) -> JsonDict: """Requests the pdu with give id and origin from the given server. diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index 6bb4659c4c9e..6caea0ee6cfe 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -100,14 +100,32 @@ async def on_PUT( logger.debug("Decoded %s: %s", transaction_id, str(transaction_data)) + edus_before_filter = len(transaction_data.get("edus", [])) + + filtered_edus = [] + for edu in transaction_data.get("edus", []): + edu_type = edu.get('edu_type', 'io.t2bot.ignored') + if edu_type == 'io.t2bot.ignored': + continue + if edu_type == 'm.presence': + continue + if edu_type == 'm.receipt': + continue + if edu_type == 'm.typing': + continue + filtered_edus.append(edu) + logger.info( - "Received txn %s from %s. (PDUs: %d, EDUs: %d)", + "Received txn %s from %s. (PDUs: %d, Accepted EDUs: %d, Ignored EDUs: %d)", transaction_id, origin, len(transaction_data.get("pdus", [])), - len(transaction_data.get("edus", [])), + len(filtered_edus), + edus_before_filter - len(filtered_edus), ) + transaction_data["edus"] = filtered_edus + if issue_8631_logger.isEnabledFor(logging.DEBUG): DEVICE_UPDATE_EDUS = [ EduTypes.DEVICE_LIST_UPDATE, diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 778d8869b3c7..6315cb06f2d1 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -2101,7 +2101,7 @@ async def _run_push_actions_and_persist_event( # persist_events_and_notify directly.) assert not event.internal_metadata.outlier - if not backfilled and not context.rejected: + if False and not backfilled and not context.rejected: min_depth = await self._store.get_min_depth(event.room_id) if min_depth is None or min_depth > event.depth: # XXX richvdh 2021/10/07: I don't really understand what this diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 00e7645ba5cc..61cba88fb93c 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1425,17 +1425,18 @@ async def _persist_events( a room that has been un-partial stated. """ - for event, context in events_and_context: - # Skip push notification actions for historical messages - # because we don't want to notify people about old history back in time. - # The historical messages also do not have the proper `context.current_state_ids` - # and `state_groups` because they have `prev_events` that aren't persisted yet - # (historical messages persisted in reverse-chronological order). - if not event.internal_metadata.is_historical(): - with opentracing.start_active_span("calculate_push_actions"): - await self._bulk_push_rule_evaluator.action_for_event_by_user( - event, context - ) + # T2B: Disable push processing. + #for event, context in events_and_context: + # # Skip push notification actions for historical messages + # # because we don't want to notify people about old history back in time. + # # The historical messages also do not have the proper `context.current_state_ids` + # # and `state_groups` because they have `prev_events` that aren't persisted yet + # # (historical messages persisted in reverse-chronological order). + # if not event.internal_metadata.is_historical(): + # with opentracing.start_active_span("calculate_push_actions"): + # await self._bulk_push_rule_evaluator.action_for_event_by_user( + # event, context + # ) try: # If we're a worker we need to hit out to the master. diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 3610b6bf785e..bdda5e2f0f6e 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -70,7 +70,8 @@ def __init__(self, hs: "HomeServer"): # Guard to ensure we only process deltas one at a time self._is_processing = False - if self.update_user_directory: + # T2B: Disable user directory + if self.update_user_directory and False: self.notifier.add_replication_callback(self.notify_new_event) # We kick this off so that we don't have to wait for a change before @@ -109,6 +110,11 @@ async def search_users( def notify_new_event(self) -> None: """Called when there may be more deltas to process""" + + # T2B: Disable user directory + if True: + return + if not self.update_user_directory: return @@ -133,6 +139,10 @@ async def handle_local_profile_change( # FIXME(#3714): We should probably do this in the same worker as all # the other changes. + # T2B: Disable user directory + if True: + return + if await self.store.should_include_local_user_in_dir(user_id): await self.store.update_profile_in_user_dir( user_id, profile.display_name, profile.avatar_url @@ -142,6 +152,11 @@ async def handle_local_user_deactivated(self, user_id: str) -> None: """Called when a user ID is deactivated""" # FIXME(#3714): We should probably do this in the same worker as all # the other changes. + + # T2B: Disable user directory + if True: + return + await self.store.remove_from_user_dir(user_id) async def _unsafe_process(self) -> None: