Add optimized view for fetching room moderators

Using the user_permissions view for this is very slow: because it has to
check each user.moderator value in the various COALESCEs it ends up
doing a full scan of the `users` table.

This adds a new `room_moderators` view for fetching moderator lists that
is a couple of orders of magnitude faster (by constructing the query
differently).

Author: jagerman

sogs/migrations/__init__.py

@@ -11,6 +11,7 @@
     new_columns,
     new_tables,
     room_accessible,
+    room_moderators,
     seqno_etc,
     user_permissions,
     user_perm_futures,
@@ -42,6 +43,7 @@ def migrate(conn, *, check_only=False):
         message_views,
         user_perm_futures,
         room_accessible,
+        room_moderators,
         user_permissions,
         file_message,
         import_hacks,

sogs/migrations/room_moderators.py

@@ -0,0 +1,99 @@
+import logging
+from .exc import DatabaseUpgradeRequired
+
+
+def migrate(conn, *, check_only):
+    """
+    Adds the room_moderators view and drops/replaces the user_permission_overrides_public_mods
+    index.
+    """
+
+    from .. import db
+
+    if 'room_moderators' in db.metadata.tables:
+        return False
+
+    logging.warning("DB migration: create room_moderators view")
+    if check_only:
+        raise DatabaseUpgradeRequired("Create room_moderators view")
+
+    if db.engine.name == "sqlite":
+        conn.execute(
+            """
+CREATE VIEW room_moderators AS
+SELECT session_id, mods.* FROM (
+    SELECT
+        room,
+        "user",
+        MAX(visible_mod) & 1 AS visible_mod,
+        MAX(admin) AS admin,
+        MAX(room_moderator) AS room_moderator,
+        MAX(global_moderator) AS global_moderator
+    FROM (
+        SELECT
+            room,
+            "user",
+            CASE WHEN visible_mod THEN 3 ELSE 2 END AS visible_mod,
+            admin,
+            TRUE AS room_moderator,
+            FALSE AS global_moderator
+        FROM user_permission_overrides WHERE moderator
+
+        UNION ALL
+
+        SELECT
+            rooms.id AS room,
+            users.id as "user",
+            CASE WHEN visible_mod THEN 1 ELSE 0 END AS visible_mod,
+            admin,
+            FALSE as room_moderator,
+            TRUE as global_moderator
+        FROM users CROSS JOIN rooms WHERE moderator
+    ) m GROUP BY "user", room
+) mods JOIN users on "user" = users.id
+"""
+        )
+    else:  # postgres
+        conn.execute(
+            """
+CREATE VIEW room_moderators AS
+SELECT session_id, mods.* FROM (
+    SELECT
+        room,
+        "user",
+        CAST(MAX(visible_mod) & 1 AS BOOLEAN) AS visible_mod,
+        bool_or(admin) AS admin,
+        bool_or(room_moderator) AS room_moderator,
+        bool_or(global_moderator) AS global_moderator
+    FROM (
+        SELECT
+            room,
+            "user",
+            CASE WHEN visible_mod THEN 3 ELSE 2 END AS visible_mod,
+            admin,
+            TRUE AS room_moderator,
+            FALSE AS global_moderator
+        FROM user_permission_overrides WHERE moderator
+
+        UNION ALL
+
+        SELECT
+            rooms.id AS room,
+            users.id as "user",
+            CASE WHEN visible_mod THEN 1 ELSE 0 END AS visible_mod,
+            admin,
+            FALSE as room_moderator,
+            TRUE as global_moderator
+        FROM users CROSS JOIN rooms WHERE moderator
+    ) m GROUP BY "user", room
+) mods JOIN users on "user" = users.id
+"""
+        )
+
+    conn.execute("DROP INDEX IF EXISTS user_permission_overrides_public_mods")
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS user_permission_overrides_mods "
+        "ON user_permission_overrides(room) WHERE moderator"
+    )
+
+    return True

sogs/model/room.py

@@ -981,11 +981,12 @@ def get_mods(self, user=None):
         ([public_mods], [public_admins], [hidden_mods], [hidden_admins])
         """
 
+        visible_clause = "" if self.check_moderator(user) else "AND visible_mod"
         m, hm, a, ha = [], [], [], []
         for session_id, visible, admin in query(
-            """
-            SELECT session_id, visible_mod, admin FROM user_permissions
-            WHERE room = :r AND moderator
+            f"""
+            SELECT session_id, visible_mod, admin FROM room_moderators
+            WHERE room = :r {visible_clause}
             ORDER BY session_id
             """,
             r=self.id,
@@ -996,9 +997,6 @@ def get_mods(self, user=None):
 
             ((a if admin else m) if visible else (ha if admin else hm)).append(session_id)
 
-        if user is None or not any(user.session_id in modlist for modlist in (m, hm, a, ha)):
-            hm, ha = [], []
-
         return m, a, hm, ha
 
     def get_all_moderators(self):

sogs/schema.pgsql

@@ -240,8 +240,7 @@ CREATE TABLE user_permission_overrides (
     PRIMARY KEY(room, "user"),
     CHECK(NOT (banned AND (moderator OR admin))) /* Mods/admins cannot be banned */
 );
-CREATE INDEX user_permission_overrides_public_mods ON
-    user_permission_overrides(room) WHERE moderator OR admin;
+CREATE INDEX user_permission_overrides_mods ON user_permission_overrides(room) WHERE moderator;
 
 -- Create a trigger to maintain the implication "admin implies moderator"
 CREATE OR REPLACE FUNCTION trigger_user_perms_admins_are_mods()
@@ -360,7 +359,8 @@ EXECUTE PROCEDURE trigger_room_metadata_pinned_remove();
 
 -- View of permissions; for users with an entry in user_permissions we use those values; for null
 -- values or no user_permissions entry we return the room's default read/write values (and false for
--- the other fields).
+-- the other fields).  This view should only be used for querying individual user permissions as it
+-- will involve a full table scan on `users` if not given a "user" value in the query.
 CREATE VIEW user_permissions AS
 SELECT
     rooms.id AS room,
@@ -388,6 +388,44 @@ FROM
     users CROSS JOIN rooms LEFT OUTER JOIN user_permission_overrides ON
         (users.id = user_permission_overrides."user" AND rooms.id = user_permission_overrides.room);
 
+-- Used to accesses the moderator list for a room.  This view is considerably faster than querying
+-- the `user_permissions` table for a list of all moderators.
+CREATE VIEW room_moderators AS
+SELECT session_id, mods.* FROM (
+    SELECT
+        room,
+        "user",
+        -- visible_mod gets priority from the per-room row if it exists, so we use 3/2 for the
+        -- per-room value below, 1/0 for the global value, take the max, then look for an odd value
+        -- to give us the visibility bit:
+        CAST(MAX(visible_mod) & 1 AS BOOLEAN) AS visible_mod,
+        bool_or(admin) AS admin,
+        bool_or(room_moderator) AS room_moderator,
+        bool_or(global_moderator) AS global_moderator
+    FROM (
+        SELECT
+            room,
+            "user",
+            CASE WHEN visible_mod THEN 3 ELSE 2 END AS visible_mod,
+            admin,
+            TRUE AS room_moderator,
+            FALSE AS global_moderator
+        FROM user_permission_overrides WHERE moderator
+
+        UNION ALL
+
+        SELECT
+            rooms.id AS room,
+            users.id as "user",
+            CASE WHEN visible_mod THEN 1 ELSE 0 END AS visible_mod,
+            admin,
+            FALSE as room_moderator,
+            TRUE as global_moderator
+        FROM users CROSS JOIN rooms WHERE moderator
+    ) m GROUP BY "user", room
+) mods JOIN users on "user" = users.id;
+
+
 -- Scheduled changes to user permissions.  For example, to implement a 2-day timeout you would set
 -- their user_permissions.write to false, then set a `write = true` entry with a +2d timestamp here.
 -- Or to implement a join delay you could set room defaults to false then insert a value here to be

sogs/schema.sqlite

@@ -215,8 +215,7 @@ CREATE TABLE user_permission_overrides (
     PRIMARY KEY(room, user),
     CHECK(NOT (banned AND (moderator OR admin))) /* Mods/admins cannot be banned */
 ) WITHOUT ROWID;
-CREATE INDEX user_permission_overrides_public_mods ON
-    user_permission_overrides(room) WHERE moderator OR admin;
+CREATE INDEX user_permission_overrides_mods ON user_permission_overrides(room) WHERE moderator;
 
 -- Create a trigger to maintain the implication "admin implies moderator"
 CREATE TRIGGER user_perms_update_admins_are_mods AFTER UPDATE OF moderator, admin ON user_permission_overrides
@@ -314,7 +313,8 @@ END;
 
 -- View of permissions; for users with an entry in user_permissions we use those values; for null
 -- values or no user_permissions entry we return the room's default read/write values (and false for
--- the other fields).
+-- the other fields).  This view should only be used for querying individual user permissions as it
+-- will involve a full table scan on `users` if not given a "user" value in the query.
 CREATE VIEW user_permissions AS
 SELECT
     rooms.id AS room,
@@ -342,6 +342,43 @@ FROM
     users JOIN rooms LEFT OUTER JOIN user_permission_overrides ON
         (users.id = user_permission_overrides.user AND rooms.id = user_permission_overrides.room);
 
+-- Used to accesses the moderator list for a room.  This view is considerably faster than querying
+-- the `user_permissions` table for a list of all moderators.
+CREATE VIEW room_moderators AS
+SELECT session_id, mods.* FROM (
+    SELECT
+        room,
+        "user",
+        -- visible_mod gets priority from the per-room row if it exists, so we use 3/2 for the
+        -- per-room value below, 1/0 for the global value, take the max, then look for an odd value
+        -- to give us the visibility bit:
+        MAX(visible_mod) & 1 AS visible_mod,
+        MAX(admin) AS admin,
+        MAX(room_moderator) AS room_moderator,
+        MAX(global_moderator) AS global_moderator
+    FROM (
+        SELECT
+            room,
+            "user",
+            CASE WHEN visible_mod THEN 3 ELSE 2 END AS visible_mod,
+            admin,
+            TRUE AS room_moderator,
+            FALSE AS global_moderator
+        FROM user_permission_overrides WHERE moderator
+
+        UNION ALL
+
+        SELECT
+            rooms.id AS room,
+            users.id as "user",
+            CASE WHEN visible_mod THEN 1 ELSE 0 END AS visible_mod,
+            admin,
+            FALSE as room_moderator,
+            TRUE as global_moderator
+        FROM users CROSS JOIN rooms WHERE moderator
+    ) m GROUP BY "user", room
+) mods JOIN users on "user" = users.id;
+
 -- Scheduled changes to user permissions.  For example, to implement a 2-day timeout you would set
 -- their user_permissions.write to false, then set a `write = true` entry with a +2d timestamp here.
 -- Or to implement a join delay you could set room defaults to false then insert a value here to be

tests/test_blinding.py

@@ -247,6 +247,9 @@ def test_blinded_transition(
 
         assert unmigrated == set()
 
+        for r in (room, room2, r3):
+            r._refresh(perms=True)
+
         # NB: "global_admin" isn't actually an admin anymore (we transferred the permission to the
         # blinded equivalent), so shouldn't see the invisible mods:
         assert room.get_mods(global_admin) == ([mod.blinded_id], [admin.blinded_id], [], [])