WIP: MSC3911: Handle media that are attached to redacted media - via quarantine

synapse/api/auth/base.py

@@ -239,6 +239,51 @@ async def check_can_change_room_list(
 
         return user_level >= send_level
 
+    async def is_moderator(self, room_id: str, requester: Requester) -> bool:
+        """Determine whether the user is moderator of the room.
+
+        Args:
+            room_id: The room_id of the room to check
+            requester: The user making the request
+        """
+        is_admin = await self.is_server_admin(requester)
+        if is_admin:
+            return True
+        await self.check_user_in_room(room_id, requester)
+
+        # We currently require the user is a "moderator" in the room. We do this
+        # by checking if they would (theoretically) be able to change the
+        # m.room.canonical_alias events
+
+        auth_events = await self._storage_controllers.state.get_current_state(
+            room_id,
+            StateFilter.from_types(
+                [
+                    POWER_KEY,
+                    CREATE_KEY,
+                ]
+            ),
+        )
+
+        send_level = event_auth.get_send_level(
+            EventTypes.CanonicalAlias,
+            "",
+            auth_events.get(POWER_KEY),
+        )
+
+        user_level = event_auth.get_user_power_level(
+            requester.user.to_string(), auth_events
+        )
+        # Check multiple moderator-level actions
+        kick_level = event_auth.get_named_level(auth_events, "kick", 50)
+        ban_level = event_auth.get_named_level(auth_events, "ban", 50)
+        redact_level = event_auth.get_named_level(auth_events, "redact", 50)
+
+        # Consider someone a moderator if they can perform key mod actions
+        moderator_threshold = min(kick_level, ban_level, redact_level)
+
+        return user_level >= send_level and user_level >= moderator_threshold
+
     @staticmethod
     def has_access_token(request: Request) -> bool:
         """Checks if the request has an access_token.

synapse/handlers/admin.py

@@ -504,6 +504,12 @@ async def _redact_all_events(
                         prev_event_ids=[event.event_id],
                         ratelimit=False,
                     )
+                    # TODO
+                    # Find all the media that was attached to the original event
+                    #  and quarantine them!
+                    await self._store.quarantine_media_by_event_id(
+                        event.event_id, requester.user.to_string()
+                    )
                 except Exception as ex:
                     logger.info(
                         "Redaction of event %s failed due to: %s", event.event_id, ex

synapse/handlers/relations.py

@@ -95,6 +95,7 @@ def __init__(self, hs: "HomeServer"):
         self._event_handler = hs.get_event_handler()
         self._event_serializer = hs.get_event_client_serializer()
         self._event_creation_handler = hs.get_event_creation_handler()
+        self._store = hs.get_datastores().main
 
     async def get_relations(
         self,
@@ -258,6 +259,12 @@ async def redact_events_related_to(
                     },
                     ratelimit=False,
                 )
+                # TODO
+                # Find all the media that was attached to the original event
+                # and quarantine them!
+                await self._store.quarantine_media_by_event_id(
+                        related_event_id, requester.user.to_string()
+                    )
             except SynapseError as e:
                 logger.warning(
                     "Failed to redact event %s (related to event %s): %s",

synapse/handlers/room_member.py

@@ -19,57 +19,59 @@
 # [This file includes modifications made by New Vector Limited]
 #
 #
 import abc
 import logging
 import random
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple
 
 from matrix_common.types.mxc_uri import MXCUri
+from synapse.api.auth.base import BaseAuth
+
 
 from synapse import types
 from synapse.api.constants import (
     AccountDataTypes,
     EventContentFields,
     EventTypes,
     GuestAccess,
     Membership,
 )
 from synapse.api.errors import (
     AuthError,
     Codes,
     PartialStateConflictError,
     ShadowBanError,
     SynapseError,
 )
 from synapse.api.ratelimiting import Ratelimiter
 from synapse.event_auth import get_named_level, get_power_level_event
 from synapse.events import EventBase, is_creator
 from synapse.events.snapshot import EventContext
 from synapse.handlers.pagination import PURGE_ROOM_ACTION_NAME
 from synapse.handlers.profile import MAX_AVATAR_URL_LEN, MAX_DISPLAYNAME_LEN
 from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler
 from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME
 from synapse.logging import opentracing
 from synapse.metrics import event_processing_positions
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http.push import ReplicationCopyPusherRestServlet
 from synapse.storage.databases.main.media_repository import LocalMedia
 from synapse.storage.databases.main.state_deltas import StateDelta
 from synapse.storage.invite_rule import InviteRule
 from synapse.types import (
     JsonDict,
     Requester,
     RoomAlias,
     RoomID,
     StateMap,
     UserID,
     create_requester,
     get_domain_from_id,
 )
 from synapse.types.state import StateFilter
 from synapse.util.async_helpers import Linearizer
 from synapse.util.distributor import user_left_room
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -882,7 +884,17 @@
                         # The actual info is irrelevant at this stage, just ignore. This
                         # will raise if the media does not exist
                         _ = await media_repo.get_media_info(existing_mxc_uri)
-
+                        # TODO: thus get_media_info no longer checkes the quarantined_by field, we need to check it here
+                        # with the requester. information.
+                        # if media.quarantined_by:
+                        #     assert isinstance(self.auth, BaseAuth)
+                        #     is_moderator = await self.auth.is_moderator(room_id, requester)
+                        #     if not is_moderator:
+                        #         raise SynapseError(
+                        #             HTTPStatus.NOT_FOUND,
+                        #             "Media not found",
+                        #             errcode=Codes.NOT_FOUND,
+                        #         )
                         new_mxc_uri = await media_repo.copy_media(
                             existing_mxc_uri, requester.user, 20_000
                         )
@@ -938,6 +950,17 @@
 
                     else:
                         media_object = await media_repo.get_media_info(new_mxc_uri)
+                        # TODO: thus get_media_info no longer checkes the quarantined_by field, we need to check it here
+                        # with the requester. information.
+                        if media_object.quarantined_by:
+                            assert isinstance(self.auth, BaseAuth)
+                            is_moderator = await self.auth.is_moderator(room_id, requester)
+                            if not is_moderator:
+                                raise SynapseError(
+                                    HTTPStatus.NOT_FOUND,
+                                    "Media not found",
+                                    errcode=Codes.NOT_FOUND,
+                                )
                         assert isinstance(media_object, LocalMedia)
                         media_info_for_attachment = {media_object}
                         content[EventContentFields.MEMBERSHIP_AVATAR_URL] = str(

synapse/media/media_repository.py

@@ -19,76 +19,78 @@
 # [This file includes modifications made by New Vector Limited]
 #
 #
 import errno
 import logging
 import os
 import shutil
 from http import HTTPStatus
 from io import BytesIO
 from typing import IO, TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union
+from synapse.api.auth.base import BaseAuth
+
 
 import attr
 from matrix_common.types.mxc_uri import MXCUri
 
 import twisted.web.http
 from twisted.internet.defer import Deferred
 
 from synapse.api.constants import EventTypes, HistoryVisibility, Membership
 from synapse.api.errors import (
     Codes,
     FederationDeniedError,
     HttpResponseException,
     NotFoundError,
     RequestSendFailed,
     SynapseError,
     UnauthorizedRequestAPICallError,
     cs_error,
 )
 from synapse.api.ratelimiting import Ratelimiter
 from synapse.config.repository import ThumbnailRequirement
 from synapse.http.server import respond_with_json
 from synapse.http.site import SynapseRequest
 from synapse.logging.context import defer_to_thread
 from synapse.logging.opentracing import trace
 from synapse.media._base import (
     FileInfo,
     Responder,
     ThumbnailInfo,
     check_for_cached_entry_and_respond,
     get_filename_from_headers,
     respond_404,
     respond_with_multipart_responder,
     respond_with_responder,
 )
 from synapse.media.filepath import MediaFilePaths
 from synapse.media.media_storage import (
     MediaStorage,
     SHA256TransparentIOReader,
     SHA256TransparentIOWriter,
 )
 from synapse.media.storage_provider import StorageProviderWrapper
 from synapse.media.thumbnailer import Thumbnailer, ThumbnailError
 from synapse.media.url_previewer import UrlPreviewer
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http.media import ReplicationCopyMediaServlet
 from synapse.storage.databases.main.media_repository import (
     LocalMedia,
     MediaRestrictions,
     RemoteMedia,
 )
 from synapse.types import JsonDict, Requester, UserID
 from synapse.types.state import StateFilter
 from synapse.util import json_decoder
 from synapse.util.async_helpers import Linearizer
 from synapse.util.retryutils import NotRetryingDestination
 from synapse.util.stringutils import random_string
 from synapse.visibility import (
     _HISTORY_VIS_KEY,
     MEMBERSHIP_PRIORITY,
     VISIBILITY_PRIORITY,
     filter_events_for_client,
     get_effective_room_visibility_from_state,
 )
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -156,10 +158,11 @@
             raise SynapseError(
                 HTTPStatus.NOT_FOUND, "Media not found", errcode=Codes.NOT_FOUND
             )
-        if media_info.quarantined_by:
-            raise SynapseError(
-                HTTPStatus.NOT_FOUND, "Media not found", errcode=Codes.NOT_FOUND
-            )
+        # TODO: We no longer check the quarantined_by here, because we need to check it with the requester.
+        # if media_info.quarantined_by:
+        #     raise SynapseError(
+        #         HTTPStatus.NOT_FOUND, "Media not found", errcode=Codes.NOT_FOUND
+        #     )
         return media_info
 
     async def create_or_update_content(
@@ -312,13 +315,13 @@
         return attachments
 
     async def is_media_visible(
-        self, requesting_user: UserID, media_info_object: Union[LocalMedia, RemoteMedia]
+        self, requester: Requester, media_info_object: Union[LocalMedia, RemoteMedia]
     ) -> None:
         """
         Verify that media requested for download should be visible to the user making
         the request
         """
-
+        requester_user_id_str = requester.user.to_string()
         if not self.enable_media_restriction:
             return
 
@@ -329,7 +332,7 @@
             # When the media has not been attached yet, only the originating user can
             # see it. But once attachments have been formed, standard other rules apply
             if isinstance(media_info_object, LocalMedia) and (
-                requesting_user.to_string() == str(media_info_object.user_id)
+                requester_user_id_str == str(media_info_object.user_id)
             ):
                 return
 
@@ -338,7 +341,7 @@
                 "Media ID ('%s') as requested by '%s' was restricted but had no "
                 "attachments",
                 media_info_object.media_id,
-                requesting_user.to_string(),
+                requester_user_id_str,
             )
             raise UnauthorizedRequestAPICallError(
                 f"Media requested ('{media_info_object.media_id}') is restricted"
@@ -363,18 +366,27 @@
                 # time to find out if a given room ever had anything other than a leave
                 # event, this is the simplest without having to do tablescans
 
+                if media_info_object.quarantined_by:
+                    assert isinstance(self.auth, BaseAuth)
+                    is_moderator = await self.auth.is_moderator(event_base.room_id, requester)
+                    if not is_moderator:
+                        raise SynapseError(
+                            HTTPStatus.NOT_FOUND, "Media not found", errcode=Codes.NOT_FOUND
+                        )
+                    return
+
                 # Need membership of NOW
                 (
                     membership_now,
                     _,
                 ) = await self.store.get_local_current_membership_for_user_in_room(
-                    requesting_user.to_string(), event_base.room_id
+                    requester_user_id_str, event_base.room_id
                 )
 
                 if not membership_now:
                     membership_now = Membership.LEAVE
 
-                membership_state_key = (EventTypes.Member, requesting_user.to_string())
+                membership_state_key = (EventTypes.Member, requester_user_id_str)
                 types = (_HISTORY_VIS_KEY, membership_state_key)
 
                 # and history visibility and membership of THEN
@@ -467,7 +479,7 @@
                 storage_controllers = self.hs.get_storage_controllers()
                 filtered_events = await filter_events_for_client(
                     storage_controllers,
-                    requesting_user.to_string(),
+                    requester_user_id_str,
                     [event_base],
                 )
                 if len(filtered_events) > 0:
@@ -487,22 +499,22 @@
             if self.hs.config.server.limit_profile_requests_to_users_who_share_rooms:
                 # First take care of the case where the requesting user IS the creating
                 # user. The other function below does not handle this.
-                if requesting_user.to_string() == attached_profile_user_id.to_string():
+                if requester_user_id_str == attached_profile_user_id.to_string():
                     return
 
                 # This call returns a set() that contains which of the "other_user_ids"
                 # share a room. Since we give it only one, if bool(set()) is True, then they
                 # share some room or had at least one invite between them.
                 if not await self.store.do_users_share_a_room_joined_or_invited(
-                    requesting_user.to_string(),
+                    requester_user_id_str,
                     [attached_profile_user_id.to_string()],
                 ):
                     logger.debug(
                         "Media ID (%s) as requested by '%s' was restricted by "
                         "profile, but was not allowed(is "
                         "'limit_profile_requests_to_users_who_share_rooms' enabled?)",
                         media_info_object.media_id,
-                        requesting_user.to_string(),
+                        requester_user_id_str,
                     )
 
                     raise UnauthorizedRequestAPICallError(
@@ -521,7 +533,7 @@
             "Media ID (%s) as requested by '%s' was restricted, but was not "
             "allowed(media_attachments=%s)",
             media_info_object.media_id,
-            requesting_user.to_string(),
+            requester_user_id_str,
             media_info_object.attachments,
         )
         raise UnauthorizedRequestAPICallError(
@@ -812,6 +824,7 @@
         logger.info("Stored local media in file %r", fname)
 
         if should_quarantine:
+            # Question: why doesn't it stop the create or update when the hash is quarantined?
             logger.warning(
                 "Media has been automatically quarantined as it matched existing quarantined media"
             )
@@ -855,6 +868,7 @@
                 user_id=auth_user,
                 sha256=sha256,
                 quarantined_by="system" if should_quarantine else None,
+                # Question: I thought `quarantined_by` should be the UserID, not something else.
                 restricted=restricted,
             )
         else:
@@ -888,6 +902,15 @@
         """
 
         old_media_info = await self.get_media_info(existing_mxc)
+
+        # TODO: thus get_media_info no longer checkes the quarantined_by field, we need to check it here
+        # If the media is quarantined, we should not copy it even though user is a moderator.
+        if old_media_info.quarantined_by:
+            raise SynapseError(
+                HTTPStatus.NOT_FOUND,
+                "Media not found",
+                errcode=Codes.NOT_FOUND,
+            )
         if isinstance(old_media_info, RemoteMedia):
             file_info = FileInfo(
                 server_name=old_media_info.media_origin,
@@ -958,10 +981,11 @@
                 respond_404(request)
                 return None
 
-            if media_info.quarantined_by:
-                logger.info("Media %s is quarantined", media_id)
-                respond_404(request)
-                return None
+            # TODO: we should not block here on quarantined media, because we want the moderators still have access to it.
+            # if media_info.quarantined_by:
+            #     logger.info("Media %s is quarantined", media_id)
+            #     respond_404(request)
+            #     return None
 
             # The file has been uploaded, so stop looping
             if media_info.media_length is not None:
@@ -1267,9 +1291,10 @@
             file_id = media_info.filesystem_id
             file_info = FileInfo(server_name, file_id)
 
-            if media_info.quarantined_by:
-                logger.info("Media is quarantined")
-                raise NotFoundError()
+            # TODO: we should not block here on quarantined media, because we want the moderators still have access to it.
+            # if media_info.quarantined_by:
+            #     logger.info("Media is quarantined")
+            #     raise NotFoundError()
 
             if not media_info.media_type:
                 media_info = attr.evolve(

synapse/rest/client/media.py

@@ -375,6 +375,11 @@ async def on_POST(
         if not media_info:
             raise NotFoundError()
         if media_info.quarantined_by:
+            # TODO:
+            # if the requester is moderator of the room
+            # allow download
+            # if not a moderator, return 404
+            # But do moderators need to copy quarantined media?
             raise NotFoundError()
 
         await self._validate_user_media_limit(requester, media_info)

synapse/rest/client/room.py

@@ -1402,7 +1402,10 @@ async def _do(
                 ) = await self.event_creation_handler.create_and_send_nonmember_event(
                     requester, event_dict, txn_id=txn_id
                 )
-
+                # TODO
+                # Find all the media that was attached to the original event
+                # and quarantine them.
+                await self._store.quarantine_media_by_event_id(event_id, requester.user.to_string())
                 if with_relations:
                     run_as_background_process(
                         "redact_related_events",

synapse/storage/databases/main/media_repository.py

@@ -19,21 +19,23 @@
 # [This file includes modifications made by New Vector Limited]
 #
 #
 import logging
 from enum import Enum
 from http import HTTPStatus
 from typing import (
     TYPE_CHECKING,
     Collection,
     Iterable,
     List,
     Optional,
     Tuple,
     Union,
     cast,
 )
 
 import attr
+from synapse.storage.engines import PostgresEngine
+import time
 
 from synapse.api.constants import Direction
 from synapse.api.errors import Codes, SynapseError