Add sogs blinded key permission migration

Adds migration of unblinded user permissions to blinded user permissions
on first login.

This works by composing a list of unmigrated ids at sogs startup: we
track the positive alternative of the two possible blinded Ed keys.
When a user first authenticates and doesn't exist, we check to see if
their |kA| is in the needs_migration table and, if it is, we construct
the new `users` row by copying over everything from the old (unblinded)
`users` row, updating all permissions and futures referencing the old id
to instead reference the new id, and then remove global settings on the
old account.

Effectively this *transfers* all permissions from the old account to the
new one, so that we aren't left with duplicates moderators/bans/etc.
anywhere.

This commit includes blinding tests, and removes the old derived key
code (which was from an obsolete, draft blinded id design).

Author: jagerman

contrib/pg-import.py

@@ -65,6 +65,7 @@
     "user_ban_futures",
     "user_request_nonces",
     "inbox",
+    "needs_blinding",
 ]
 
 with pgsql.transaction():

sogs/crypto.py

@@ -6,16 +6,14 @@
 from nacl.public import PrivateKey
 from nacl.signing import SigningKey, VerifyKey
 from nacl.encoding import Base64Encoder, HexEncoder
-from nacl.bindings import crypto_scalarmult
+import nacl.bindings as sodium
 
 
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey
 
-from .utils import decode_hex_or_b64
 from .hashing import blake2b
 
-import binascii
 import secrets
 import hmac
 import functools
@@ -93,15 +91,63 @@ def server_encrypt(pk, data):
 xed25519_verify = pyonionreq.xed25519.verify
 xed25519_pubkey = pyonionreq.xed25519.pubkey
 
+# AKA "k" for blinding crypto:
+blinding_factor = sodium.crypto_core_ed25519_scalar_reduce(
+    blake2b(server_pubkey_bytes, digest_size=64)
+)
+
 
 @functools.lru_cache(maxsize=1024)
-def compute_derived_key_bytes(pk_bytes):
-    """compute derived key as bytes with no prefix"""
-    return crypto_scalarmult(server_pubkey_hash_bytes, pk_bytes)
+def compute_blinded_abs_key(x_pk: bytes, *, k: bytes = blinding_factor):
+    """
+    Computes the *positive* blinded Ed25519 pubkey from an unprefixed session X25519 pubkey (i.e. 32
+    bytes).  The returned value will always have the sign bit (i.e. the most significant bit of the
+    last byte) set to 0; the actual derived key associated with this session id could have either
+    sign.
+
+    Input and result are in bytes, without the 0x05 or 0x15 prefix.
+
+    k allows you to compute for an alternative blinding factor, but should normally be omitted.
+    """
+    A = xed25519_pubkey(x_pk)
+    kA = sodium.crypto_scalarmult_ed25519_noclamp(k, A)
 
+    if kA[31] & 0x80:
+        return kA[0:31] + bytes([kA[31] & 0x7F])
+    return kA
+
+
+def compute_blinded_abs_id(session_id: str, *, k: bytes = blinding_factor):
+    """
+    Computes the *positive* blinded id, as hex, from a prefixed, hex session id.  This function is a
+    wrapper around compute_derived_key_bytes that handles prefixes and hex conversions.
+
+    k allows you to compute for an alternative blinding factor, but should normally be omitted.
+    """
+    return '15' + compute_blinded_abs_key(bytes.fromhex(session_id[2:]), k=k).hex()
+
+
+def blinded_abs(blinded_id: str):
+    """
+    Takes a blinded hex pubkey (i.e. length 66, prefixed with 15) and returns the positive pubkey
+    alternative: that is, if the pubkey is already positive, it is returned as-is; otherwise the
+    returned value is a copy with the sign bit cleared.
+    """
+
+    # Sign bit is the MSB of the last byte, which will be at [31] of the private key, hence 64 is
+    # the most significant nibble once we convert to hex and add 2 for the prefix:
+    msn = int(blinded_id[64], 16)
+    if msn & 0x8:
+        return blinded_id[0:64] + str(msn & 0x7) + blinded_id[65:]
+    return blinded_id
+
+
+def blinded_neg(blinded_id: str):
+    """
+    Counterpart to blinded_abs that always returns the *negative* pubkey alternative.
+    """
 
-def compute_derived_id(session_id, prefix='15'):
-    """compute derived session"""
-    return prefix + binascii.hexlify(
-        compute_derived_key_bytes(decode_hex_or_b64(session_id[2:], 32))
-    ).decode('ascii')
+    msn = int(blinded_id[64], 16)
+    if msn & 0x8:
+        return blinded_id
+    return blinded_id[0:64] + f"{msn | 0x8:x}" + blinded_id[65:]

sogs/db.py

@@ -174,6 +174,8 @@ def database_init(create=None, upgrade=True):
     # Make sure the system admin users exists
     create_admin_user(conn)
 
+    check_needs_blinding(conn)
+
     return created or migrated
 
 
@@ -195,6 +197,36 @@ def create_admin_user(dbconn):
     )
 
 
+def check_needs_blinding(dbconn):
+    if not config.REQUIRE_BLIND_KEYS:
+        return
+
+    with transaction(dbconn):
+        for uid, sid in query(
+            """
+            SELECT id, session_id FROM users WHERE id IN (
+                SELECT "user" FROM user_permission_overrides
+                UNION
+                SELECT "user" FROM user_permission_futures
+                UNION
+                SELECT "user" FROM user_ban_futures
+                UNION
+                SELECT id FROM users WHERE session_id LIKE '05%' AND (admin OR moderator OR banned)
+                EXCEPT
+                SELECT "user" FROM needs_blinding
+            )
+            """,
+            dbconn=dbconn,
+        ):
+            pos_derived = crypto.compute_blinded_abs_id(sid)
+            query(
+                'INSERT INTO needs_blinding (blinded_abs, "user") VALUES (:blinded, :uid)',
+                blinded=pos_derived,
+                uid=uid,
+                dbconn=dbconn,
+            )
+
+
 engine, engine_initial_pid, metadata = None, None, None
 
 

sogs/migrations/new_tables.py

@@ -52,6 +52,22 @@
     expiry FLOAT DEFAULT (extract(epoch from now() + '15 days'))
 );
 CREATE INDEX inbox_recipient ON inbox(recipient);
+""",
+    },
+    'needs_blinding': {
+        'sqlite': [
+            """
+CREATE TABLE needs_blinding (
+    blinded_abs TEXT NOT NULL PRIMARY KEY,
+    "user" BIGINT NOT NULL UNIQUE REFERENCES users ON DELETE CASCADE
+)
+"""
+        ],
+        'pgsql': """
+CREATE TABLE needs_blinding (
+    blinded_abs TEXT NOT NULL PRIMARY KEY,
+    "user" BIGINT NOT NULL UNIQUE REFERENCES users ON DELETE CASCADE
+)
 """,
     },
 }

sogs/model/user.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from .. import crypto, db
+from .. import crypto, db, config
 from ..db import query
 from ..web import app
 from .exc import NoSuchUser, BadPermission
@@ -28,9 +28,10 @@ def __init__(self, row=None, *, id=None, session_id=None, autovivify=True, touch
         """
         Constructs a user from a pre-retrieved row *or* a session id or user primary key value.
 
-        autovivify - if True and we are given a session_id that doesn't exist, create a default user
-        row and use it to populate the object.  This is the default behaviour.  If False and the
-        session_id doesn't exist then a NoSuchUser is raised if the session id doesn't exist.
+        autovivify - if True and we are given a session_id that doesn't exist, either consider
+        importing from a pre-blinding user (if needed) or create a default user row and use it to
+        populate the object.  This is the default behaviour.  If False and the session_id doesn't
+        exist then a NoSuchUser is raised if the session id doesn't exist.
 
         touch - if True (default is False) then update the last_activity time of this user before
         returning it.
@@ -56,11 +57,15 @@ def _refresh(self, *, row=None, id=None, session_id=None, autovivify=True):
             row = query("SELECT * FROM users WHERE session_id = :s", s=session_id).first()
 
             if not row and autovivify:
-                with db.transaction():
-                    query("INSERT INTO users (session_id) VALUES (:s)", s=session_id)
-                    row = query("SELECT * FROM users WHERE session_id = :s", s=session_id).first()
-                # No need to re-touch this user since we just created them:
-                self._touched = True
+                if config.REQUIRE_BLIND_KEYS:
+                    row = self._import_blinded(session_id)
+
+                if not row:
+                    row = db.insert_and_get_row(
+                        "INSERT INTO users (session_id) VALUES (:s)", "users", "id", s=session_id
+                    )
+                    # No need to re-touch this user since we just created them:
+                    self._touched = True
 
         elif id is not None:
             row = query("SELECT * FROM users WHERE id = :u", u=id).fetchone()
@@ -75,6 +80,64 @@ def _refresh(self, *, row=None, id=None, session_id=None, autovivify=True):
             bool(row[c]) for c in ('banned', 'moderator', 'admin', 'visible_mod')
         )
 
+    def _import_blinded(self, session_id):
+        """
+        Attempts to import the user and permission rows from an unblinded session_id to a new,
+        blinded session_id row.
+
+        Any permissions/bans are *moved* from the old, unblinded id to the new blinded user record.
+        """
+
+        if not session_id.startswith('15'):
+            return
+        blind_abs = crypto.blinded_abs(session_id.lower())
+        with db.transaction():
+            to_import = query(
+                """
+                SELECT * FROM users WHERE id = (
+                    SELECT "user" FROM needs_blinding WHERE blinded_abs = :ba
+                )
+                """,
+                ba=blind_abs,
+            ).fetchone()
+
+            if to_import is None:
+                return False
+
+            row = db.insert_and_get_row(
+                """
+                INSERT INTO users
+                    (session_id, created, last_active, banned, moderator, admin, visible_mod)
+                VALUES (:sid, :cr, :la, :ban, :mod, :admin, :vis)
+                """,
+                "users",
+                "id",
+                sid=session_id,
+                cr=to_import["created"],
+                la=to_import["last_active"],
+                ban=to_import["banned"],
+                mod=to_import["moderator"],
+                admin=to_import["admin"],
+                vis=to_import["visible_mod"],
+            )
+            # If we have any global ban/admin/mod then clear them (because we've just set up the
+            # global ban/mod/admin permissions for the blinded id in the query above).
+            query(
+                "UPDATE users SET banned = FALSE, admin = FALSE, moderator = FALSE WHERE id = :u",
+                u=to_import["id"],
+            )
+
+            for t in ("user_permission_overrides", "user_permission_futures", "user_ban_futures"):
+                query(
+                    f'UPDATE {t} SET "user" = :new WHERE "user" = :old',
+                    new=row["id"],
+                    old=to_import["id"],
+                )
+
+            query('DELETE FROM needs_blinding WHERE "user" = :u', u=to_import["id"])
+
+            return row
+
     def __str__(self):
         """Returns string representation of a user: U[050123…cdef], the id prefixed with @ or % if
         the user is a global admin or moderator, respectively."""
@@ -248,13 +311,6 @@ def system_user(self):
         created for internal database tasks"""
         return self.session_id[0:2] == "ff" and self.session_id[2:] == crypto.server_pubkey_hex
 
-    @property
-    def derived_key(self):
-        """get the derived key for this user"""
-        if self.session_id[0:2] == '15':
-            return self.session_id
-        return crypto.compute_derived_id(self.session_id)
-
 
 class SystemUser(User):
     """

sogs/schema.pgsql

@@ -172,6 +172,17 @@ FOR EACH ROW WHEN (NEW.admin AND NOT NEW.moderator)
 EXECUTE PROCEDURE trigger_user_admins_are_mods();
 
 
+-- This table tracks unblinded session ids in user_permission (and related) rows that need to be
+-- blinded, which will happen the first time the user authenticates with their blinded id (until
+-- they do, we can't know the actual sign bit of their blinded id).  It is populated at startup
+-- when blinding is first enabled, and is used both for the initial blinding transition and when
+-- ids are added by raw session ID (e.g. when adding a moderator by session id).
+CREATE TABLE needs_blinding (
+    blinded_abs TEXT NOT NULL PRIMARY KEY, -- the positive of the possible two blinded keys
+    "user" BIGINT NOT NULL UNIQUE REFERENCES users ON DELETE CASCADE
+);
+
+
 -- Effectively the same as `messages` except that it also includes the `session_id` from the users
 -- table of the user who posted it, and the session id of the whisper recipient (as `whisper_to`) if
 -- a directed whisper.

sogs/schema.sqlite

@@ -151,6 +151,18 @@ BEGIN
 END;
 
 
+-- This table tracks unblinded session ids in user_permission (and related) rows that need to be
+-- blinded, which will happen the first time the user authenticates with their blinded id (until
+-- they do, we can't know the actual sign bit of their blinded id).  It is populated at startup
+-- when blinding is first enabled, and is used both for the initial blinding transition and when
+-- ids are added by raw session ID (e.g. when adding a moderator by session id).
+CREATE TABLE needs_blinding (
+    blinded_abs TEXT NOT NULL PRIMARY KEY, -- the positive of the possible two blinded keys
+    "user" INTEGER NOT NULL UNIQUE REFERENCES users ON DELETE CASCADE
+);
+
+
+
 -- Effectively the same as `messages` except that it also includes the `session_id` from the users
 -- table of the user who posted it, and the session id of the whisper recipient (as `whisper_to`) if
 -- a directed whisper.