Merge commit '5bf8e5f55' into anoa/dinsic_release_1_21_x

* commit '5bf8e5f55':
  Convert the well known resolver to async (#8214)
  Convert additional databases to async/await part 2 (#8200)
  Make MultiWriterIDGenerator work for streams that use negative stream IDs (#8203)
  Do not install setuptools 50.0. (#8212)
  Move and rename `get_devices_with_keys_by_user` (#8204)
  Rename `get_e2e_device_keys` to better reflect its purpose (#8205)
  Add a comment about _LimitedHostnameResolver

Author: anoadragon453

INSTALL.md

@@ -73,7 +73,7 @@ mkdir -p ~/synapse
 virtualenv -p python3 ~/synapse/env
 source ~/synapse/env/bin/activate
 pip install --upgrade pip
-pip install --upgrade setuptools
+pip install --upgrade setuptools!=50.0  # setuptools==50.0 fails on some older Python versions
 pip install matrix-synapse
 ```
 

changelog.d/8200.misc

@@ -0,0 +1 @@
+Convert various parts of the codebase to async/await.

changelog.d/8203.misc

@@ -0,0 +1 @@
+Make `MultiWriterIDGenerator` work for streams that use negative values.

changelog.d/8204.misc

@@ -0,0 +1 @@
+Refactor queries for device keys and cross-signatures.

changelog.d/8205.misc

@@ -0,0 +1 @@
+ Refactor queries for device keys and cross-signatures.

changelog.d/8212.bugfix

@@ -0,0 +1 @@
+Do not install setuptools 50.0. It can lead to a broken configuration on some older Python versions.

changelog.d/8214.misc

@@ -0,0 +1 @@
+ Convert various parts of the codebase to async/await.

mypy.ini

@@ -28,6 +28,7 @@ files =
   synapse/handlers/saml_handler.py,
   synapse/handlers/sync.py,
   synapse/handlers/ui_auth,
+  synapse/http/federation/well_known_resolver.py,
   synapse/http/server.py,
   synapse/http/site.py,
   synapse/logging/,

synapse/app/_base.py

@@ -334,6 +334,13 @@ def install_dns_limiter(reactor, max_dns_requests_in_flight=100):
     This is to workaround https://twistedmatrix.com/trac/ticket/9620, where we
     can run out of file descriptors and infinite loop if we attempt to do too
     many DNS queries at once
+
+    XXX: I'm confused by this. reactor.nameResolver does not use twisted.names unless
+    you explicitly install twisted.names as the resolver; rather it uses a GAIResolver
+    backed by the reactor's default threadpool (which is limited to 10 threads). So
+    (a) I don't understand why twisted ticket 9620 is relevant, and (b) I don't
+    understand why we would run out of FDs if we did too many lookups at once.
+    -- richvdh 2020/08/29
     """
     new_resolver = _LimitedHostnameResolver(
         reactor.nameResolver, max_dns_requests_in_flight

synapse/events/builder.py

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Optional
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import attr
 from nacl.signing import SigningKey
@@ -97,14 +97,14 @@ def state_key(self):
     def is_state(self):
         return self._state_key is not None
 
-    async def build(self, prev_event_ids):
+    async def build(self, prev_event_ids: List[str]) -> EventBase:
         """Transform into a fully signed and hashed event
 
         Args:
-            prev_event_ids (list[str]): The event IDs to use as the prev events
+            prev_event_ids: The event IDs to use as the prev events
 
         Returns:
-            FrozenEvent
+            The signed and hashed event.
         """
 
         state_ids = await self._state.get_current_state_ids(
@@ -114,8 +114,13 @@ async def build(self, prev_event_ids):
 
         format_version = self.room_version.event_format
         if format_version == EventFormatVersions.V1:
-            auth_events = await self._store.add_event_hashes(auth_ids)
-            prev_events = await self._store.add_event_hashes(prev_event_ids)
+            # The types of auth/prev events changes between event versions.
+            auth_events = await self._store.add_event_hashes(
+                auth_ids
+            )  # type: Union[List[str], List[Tuple[str, Dict[str, str]]]]
+            prev_events = await self._store.add_event_hashes(
+                prev_event_ids
+            )  # type: Union[List[str], List[Tuple[str, Dict[str, str]]]]
         else:
             auth_events = auth_ids
             prev_events = prev_event_ids
@@ -138,7 +143,7 @@ async def build(self, prev_event_ids):
             "unsigned": self.unsigned,
             "depth": depth,
             "prev_state": [],
-        }
+        }  # type: Dict[str, Any]
 
         if self.is_state():
             event_dict["state_key"] = self._state_key