Move DB migration code out of sogs.db

`sogs.db` was getting messy with all the migration code; this moves it
all into multiple files in `sogs.migrations.*`, where each migration
submodule has a `migrate(conn)` function to perform a migration.

Aside from moving all the migrations, this also renames/updates the
existing `sogs.migrate01x` to `sogs.migrations.v_0_1_x` for consistency.

Author: jagerman

sogs/db.py

@@ -0,0 +1,55 @@
+import logging
+import coloredlogs
+
+from .. import config
+
+from . import (
+    import_hacks,
+    message_details_deleter,
+    message_views,
+    new_columns,
+    new_tables,
+    seqno_etc,
+    user_perm_futures,
+    v_0_1_x,
+)
+
+logger = logging.getLogger(__name__)
+coloredlogs.install(milliseconds=True, isatty=True, logger=logger, level=config.LOG_LEVEL)
+
+
+def migrate(conn):
+    """Perform database migrations/updates/etc."""
+
+    from .. import db
+
+    # NB: migration order here matters; some later migrations require earlier migrations
+    for migration in (
+        v_0_1_x,
+        new_tables,
+        new_columns,
+        message_details_deleter,
+        import_hacks,
+        seqno_etc,
+        message_views,
+        user_perm_futures,
+    ):
+        changes = False
+        with db.transaction(conn):
+            changes = migration.migrate(conn)
+        if changes:
+            db.metadata.clear()
+            db.metadata.reflect(bind=db.engine, views=True)
+
+
+#        migrate_v01x,
+#        add_new_tables,
+#        add_new_columns,
+#        create_message_details_deleter,
+#        check_for_hacks,
+#        seqno_etc_updates,
+#        update_message_views,
+#        user_perm_future_updates,
+
+#        add_accessible_perm_bit,
+#    ):

sogs/migrations/__init__.py

@@ -0,0 +1,39 @@
+import logging
+
+
+def migrate(conn):
+    """
+    The 0.1.x migration sets up a file_id_hacks table to map old ids to new ids; if it's present and
+    non-empty then we enable "hack" mode.  (This should empty out over 15 days as attachments
+    expire).
+
+    We also have a room_import_hacks table that lets us map old message ids to new ids (because in
+    the old database message ids overlapped, but in the new database they are unique).  The consists
+    of a max id and an offset that lets us figure out the new (current database) id.  For instance,
+    some range of messages in room xyz with old ids [1,5000] could get inserted as ids [4321, 9320],
+    so max would be 5000 and offset would be 4320: old message id 3333 will have new message id
+    3333+4320 = 7653.  We read all the offsets once at startup and stash them in ROOM_IMPORT_HACKS.
+    """
+
+    from .. import db
+
+    if 'file_id_hacks' in db.metadata.tables:
+        # If the table exists but is empty (i.e. because all the attachments expired) then we should
+        # drop it.
+        n_fid_hacks = conn.execute("SELECT COUNT(*) FROM file_id_hacks").first()[0]
+        if n_fid_hacks == 0:
+            logging.warning("Dropping file_id_hacks old sogs import table (no longer required)")
+            db.metadata.tables['file_id_hacks'].drop(db.engine)
+        else:
+            logging.warning("Keeping file_id_hacks old sogs import table (still required)")
+            global HAVE_FILE_ID_HACKS
+            db.HAVE_FILE_ID_HACKS = True
+
+    try:
+        rows = conn.execute(
+            "SELECT room, old_message_id_max, message_id_offset FROM room_import_hacks"
+        )
+        for (room, id_max, offset) in rows:
+            db.ROOM_IMPORT_HACKS[room] = (id_max, offset)
+    except Exception:
+        pass

sogs/migrations/import_hacks.py

@@ -0,0 +1,17 @@
+def migrate(conn):
+
+    from .. import db
+
+    if db.engine.name == "sqlite":
+        conn.execute(
+            """
+CREATE TRIGGER IF NOT EXISTS message_details_deleter INSTEAD OF DELETE ON message_details
+FOR EACH ROW WHEN OLD.data IS NOT NULL
+BEGIN
+    UPDATE messages SET data = NULL, data_size = NULL, signature = NULL
+        WHERE id = OLD.id;
+END
+"""
+        )
+
+    return False  # No need to refresh metadata even if we added the trigger above.

sogs/migrations/message_details_deleter.py

@@ -0,0 +1,46 @@
+import logging
+
+
+def migrate(conn):
+    from .. import db
+
+    if db.engine.name == "sqlite":
+        if any(
+            x not in db.metadata.tables['message_metadata'].c for x in ('whisper_to', 'filtered')
+        ):
+            logging.warning("DB migration: replacing message_metadata/message_details views")
+            conn.execute("DROP VIEW IF EXISTS message_metadata")
+            conn.execute("DROP VIEW IF EXISTS message_details")
+            conn.execute(
+                """
+CREATE VIEW message_details AS
+SELECT messages.*, uposter.session_id, uwhisper.session_id AS whisper_to
+    FROM messages
+        JOIN users uposter ON messages."user" = uposter.id
+        LEFT JOIN users uwhisper ON messages.whisper = uwhisper.id
+"""
+            )
+            conn.execute(
+                """
+CREATE TRIGGER message_details_deleter INSTEAD OF DELETE ON message_details
+FOR EACH ROW WHEN OLD.data IS NOT NULL
+BEGIN
+    UPDATE messages SET data = NULL, data_size = NULL, signature = NULL
+        WHERE id = OLD.id;
+END
+"""
+            )
+            conn.execute(
+                """
+CREATE VIEW message_metadata AS
+SELECT id, room, "user", session_id, posted, edited, seqno, filtered, whisper_to,
+        length(data) AS data_unpadded, data_size, length(signature) as signature_length
+    FROM message_details
+"""
+            )
+
+            return True
+
+    # else: don't worry about this for postgresql because initial pg support had the fix
+
+    return False

sogs/migrations/message_views.py

@@ -0,0 +1,29 @@
+import logging
+
+
+def migrate(conn):
+    """
+    New columns that might need to be added that don't require more complex migrations beyond simply
+    adding the column.
+    """
+
+    from .. import db
+
+    new_table_cols = {
+        'messages': {
+            'whisper': 'INTEGER REFERENCES users(id)',
+            'whisper_mods': 'BOOLEAN NOT NULL DEFAULT FALSE',
+            'filtered': 'BOOLEAN NOT NULL DEFAULT FALSE',
+        }
+    }
+
+    added = False
+
+    for table, cols in new_table_cols.items():
+        for name, definition in cols.items():
+            if name not in db.metadata.tables[table].c:
+                logging.warning(f"DB migration: Adding new column {table}.{name}")
+                conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {definition}")
+                added = True
+
+    return added

sogs/migrations/new_columns.py

@@ -0,0 +1,80 @@
+import logging
+
+
+# { table_name => { 'sqlite': ['query1', 'query2'], 'pgsql': "query1; query2" } }
+table_creations = {
+    'user_request_nonces': {
+        'sqlite': [
+            """
+CREATE TABLE user_request_nonces (
+    user INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    nonce BLOB NOT NULL UNIQUE,
+    expiry FLOAT NOT NULL DEFAULT ((julianday('now') - 2440587.5 + 1.0)*86400.0) /* now + 24h */
+)
+""",
+            """
+CREATE INDEX user_request_nonces_expiry ON user_request_nonces(expiry)
+""",
+        ],
+        'pgsql': """
+CREATE TABLE user_request_nonces (
+    "user" BIGINT NOT NULL REFERENCES users ON DELETE CASCADE,
+    nonce BYTEA NOT NULL UNIQUE,
+    expiry FLOAT NOT NULL DEFAULT (extract(epoch from now() + '24 hours'))
+);
+CREATE INDEX user_request_nonces_expiry ON user_request_nonces(expiry)
+""",
+    },
+    'inbox': {
+        'sqlite': [
+            """
+CREATE TABLE inbox (
+    id INTEGER PRIMARY KEY,
+    recipient INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    sender INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    body BLOB NOT NULL,
+    posted_at FLOAT DEFAULT ((julianday('now') - 2440587.5)*86400.0),
+    expiry FLOAT DEFAULT ((julianday('now') - 2440587.5 + 1.0)*86400.0) /* now + 24h */
+)
+""",
+            """
+CREATE INDEX inbox_recipient ON inbox(recipient)
+""",
+        ],
+        'pgsql': """
+CREATE TABLE inbox (
+    id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    recipient BIGINT NOT NULL REFERENCES users ON DELETE CASCADE,
+    sender BIGINT NOT NULL REFERENCES users ON DELETE CASCADE,
+    body BYTEA NOT NULL,
+    posted_at FLOAT DEFAULT (extract(epoch from now())),
+    expiry FLOAT DEFAULT (extract(epoch from now() + '15 days'))
+);
+CREATE INDEX inbox_recipient ON inbox(recipient);
+""",
+    },
+}
+
+
+def migrate(conn):
+    """Adds new tables that don't have any special migration requirement beyond creation"""
+
+    from .. import db
+
+    added = False
+
+    for table, v in table_creations.items():
+        if table in db.metadata.tables:
+            continue
+
+        logging.warning(f"DB migration: Adding new table {table}")
+
+        if db.engine.name == 'sqlite':
+            for query in v['sqlite']:
+                conn.execute(query)
+        else:
+            conn.execute(v['pgsql'])
+
+        added = True
+
+    return added