diff --git a/conftest.py b/conftest.py index e7343774..9c3eeaf2 100644 --- a/conftest.py +++ b/conftest.py @@ -267,7 +267,7 @@ def blind_user2(db): @pytest.fixture def no_rate_limit(): """Disables post rate limiting for the test""" - import sogs.model.room as mroom + from sogs.model.room import Room as mroom saved = (mroom.rate_limit_size, mroom.rate_limit_interval) mroom.rate_limit_size, mroom.rate_limit_interval = None, None diff --git a/contrib/auth-example.py b/contrib/auth-example.py index 0e6767b8..ed52376a 100755 --- a/contrib/auth-example.py +++ b/contrib/auth-example.py @@ -71,7 +71,6 @@ def get_signing_headers( body, blinded: bool = True, ): - assert len(server_pk) == 32 assert len(nonce) == 16 diff --git a/contrib/pg-import.py b/contrib/pg-import.py index 61c7a1a3..df832ae8 100755 --- a/contrib/pg-import.py +++ b/contrib/pg-import.py @@ -86,7 +86,6 @@ with pgsql.transaction(): - curin = old.cursor() curout = pgsql.cursor() @@ -131,7 +130,6 @@ curout.execute("ALTER TABLE rooms DROP CONSTRAINT room_image_fk") def copy(table): - cols = [r['name'] for r in curin.execute(f"PRAGMA table_info({table})")] if not cols: raise RuntimeError(f"Expected table {table} does not exist in sqlite db") diff --git a/contrib/uwsgi-sogs-standalone.ini b/contrib/uwsgi-sogs-standalone.ini index 16a7f977..6bd1eef2 100644 --- a/contrib/uwsgi-sogs-standalone.ini +++ b/contrib/uwsgi-sogs-standalone.ini @@ -27,7 +27,7 @@ processes = 2 enable-threads = true http = :80 mount = /=sogs.web:app -mule = sogs.mule:run +mule = @(call://sogs.mule.Mule) log-4xx = true log-5xx = true disable-logging = true diff --git a/sogs/__main__.py b/sogs/__main__.py index d897949e..dc027c0b 100644 --- a/sogs/__main__.py +++ b/sogs/__main__.py @@ -400,7 +400,6 @@ def parse_and_set_perm_flags(flags, perm_setting): sys.exit(2) elif update_room: - rooms = [] all_rooms = False global_rooms = False @@ -577,8 +576,7 @@ def parse_and_set_perm_flags(flags, perm_setting): if args.name is not None: if global_rooms or all_rooms: print( - "Error: --rooms cannot be '+' or '*' (i.e. global/all) with --name", - file=sys.stderr, + "Error: --rooms cannot be '+' or '*' (i.e. global/all) with --name", file=sys.stderr ) sys.exit(1) diff --git a/sogs/migrations/file_message.py b/sogs/migrations/file_message.py index 14eac2e5..d8680644 100644 --- a/sogs/migrations/file_message.py +++ b/sogs/migrations/file_message.py @@ -3,7 +3,6 @@ def migrate(conn, *, check_only): - from .. import db fix_fk = False diff --git a/sogs/migrations/import_hacks.py b/sogs/migrations/import_hacks.py index 1556cb70..2db9d46a 100644 --- a/sogs/migrations/import_hacks.py +++ b/sogs/migrations/import_hacks.py @@ -43,7 +43,7 @@ def migrate(conn, *, check_only): rows = conn.execute( "SELECT room, old_message_id_max, message_id_offset FROM room_import_hacks" ) - for (room, id_max, offset) in rows: + for room, id_max, offset in rows: db.ROOM_IMPORT_HACKS[room] = (id_max, offset) if not db.HAVE_FILE_ID_HACKS and 'room_import_hacks' not in db.metadata.tables: diff --git a/sogs/migrations/reactions.py b/sogs/migrations/reactions.py index acff1aa7..32acecda 100644 --- a/sogs/migrations/reactions.py +++ b/sogs/migrations/reactions.py @@ -3,7 +3,6 @@ def migrate(conn, *, check_only): - from .. import db if 'user_reactions' in db.metadata.tables: @@ -177,7 +176,6 @@ def migrate(conn, *, check_only): ) else: # postgresql - if 'seqno_data' not in db.metadata.tables['messages'].c: conn.execute( """ diff --git a/sogs/migrations/v_0_1_x.py b/sogs/migrations/v_0_1_x.py index b59c1ef3..740af695 100644 --- a/sogs/migrations/v_0_1_x.py +++ b/sogs/migrations/v_0_1_x.py @@ -39,7 +39,6 @@ def sqlite_connect_readonly(path): def import_from_0_1_x(conn): - from .. import config, db, utils # Old database database.db is a single table database containing just the list of rooms: @@ -109,7 +108,6 @@ def ins_user(session_id): ) with sqlite_connect_readonly(room_db_path) as rconn: - # Messages were stored in this: # # CREATE TABLE IF NOT EXISTS messages ( @@ -235,7 +233,6 @@ def ins_user(session_id): and data in (None, "deleted") and signature in (None, "deleted") ): - # Deleted message; we still need to insert a tombstone for it, and copy the # deletion id as the "seqno" field. (We do this with a second query # because the first query is going to trigger an automatic update of the @@ -340,7 +337,6 @@ def ins_user(session_id): n_files = rconn.execute("SELECT COUNT(*) FROM files").fetchone()[0] for file_id, timestamp in rconn.execute("SELECT id, timestamp FROM files"): - # file_id is an integer value but stored in a TEXT field, of course. file_id = int(file_id) @@ -507,7 +503,6 @@ def ins_user(session_id): """, (import_cutoff,), ): - ins_user(session_id) db.query( """ diff --git a/sogs/model/clientmanager.py b/sogs/model/clientmanager.py new file mode 100644 index 00000000..49f7a048 --- /dev/null +++ b/sogs/model/clientmanager.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import oxenmq + +from .. import crypto, db, config +from ..db import query +from ..web import app +from .exc import BadPermission, PostRateLimited +from .. import utils +from .user import User +from ..omq import omq_global +from .room import Room +from .exc import InvalidData +import heapq + +from dataclasses import dataclass, field +from typing import Optional, List, Union, Any +import time + + +""" +Complications: + - given captcha bot + - new user (not approved) posts message + - we need bot to reply with whisper to that user with simple problem + - what does the bot do with the message they tried to send? + - can store locally + - user sends reply + - bot inserts it into room (?) + +Control Flow: + 1) message comes in HTTP request + 2) unpacked/parsed/verified/permissions checked + 3) comes into relevant route (ex: add_post()) + 4) sends off to mule to be handled by bots + 5) mule has ordered list of bots by priority + 6) mule passes message to bots, which have fixed return values (insert, do not insert) + 7) if all bots approve, mule replies to worker with go ahead or vice versa for no go +""" + + +@dataclass(order=True) +class PriorityTuple(tuple): + priority: int + item: Any = field(compare=False) + + +# Simple "priority queue" of bots implemented using a dict with heap +# invariance maintained by qheap algorithm +# TODO: when bots are designed basically, add methods for polling them +# and receiving their judgments +class BotQueue: + def __init__(self) -> None: + self.queue = {} + + def _qsize(self) -> int: + return len(self.queue.keys()) + + def _empty(self) -> bool: + return not self._qsize() + + def _peek(self, priority: int): + return self.queue.get(priority) + + def _put(self, item: PriorityTuple): + temp = list(self.queue.items()) + heapq.heappush(temp, item) + self.queue = dict(temp) + + def _get(self): + return heapq.heappop(self.queue) + + +class ClientManager: + """ + Class representing an interface that manages active bots + + Object Properties: + queue - BotQueue object + """ + + def __init__(self, *, id: Optional[int] = None) -> None: + self.id = id + self.filter = False + self.bqueue = BotQueue() + self.clients = [] + + def bqempty(self): + return not self.bqueue._empty() + + def register_client(self, conn_id, cid, authlevel, bot, priority): + if not bot: + # add client to self.clients + return + + if not priority: + # if no priority is given, lowest priority is assigned + priority = self.qsize() + else: + # if priority is already taken, find next lowest + while self.queue.get(priority): + priority += 1 + self.bqueue._put(PriorityTuple(priority, bot)) + + def deregister_client(self, cid, bot): + if not bot: + # remove client from clients list + return + + # remove bot from bot queue + + def peek(self, priority: int): + return self.bqueue._peek(priority) + + def check_permission_for( + self, + room: Room, + user: Optional[User] = None, + *, + admin=False, + moderator=False, + read=False, + accessible=False, + write=False, + upload=False, + ): + """ + Checks whether `user` has the required permissions for this room and isn't banned. Returns + True if the user satisfies the permissions, False otherwise. If no user is provided then + permissions are checked against the room's defaults. + + Looked up permissions are cached within the Room instance so that looking up the same user + multiple times (i.e. from multiple parts of the code) does not re-query the database. + + Named arguments are as follows: + - admin -- if true then the user must have admin access to the room + - moderator -- if true then the user must have moderator (or admin) access to the room + - read -- if true then the user must have read access + - accessible -- if true then the user must have accessible access; note that this permission + is satisfied by *either* the `accessible` or `read` database flags (that is: read implies + accessible). + - write -- if true then the user must have write access + - upload -- if true then the user must have upload access; this should usually be combined + with write=True. + + You can specify multiple permissions as True, in which case all must be satisfied. If you + specify no permissions as required then the check only checks whether a user is banned but + otherwise requires no specific permission. + """ + + if user is None: + is_banned, can_read, can_access, can_write, can_upload, is_mod, is_admin = ( + False, + bool(room.default_read), + bool(room.default_accessible), + bool(room.default_write), + bool(room.default_upload), + False, + False, + ) + else: + if user.id not in self._perm_cache: + row = query( + """ + SELECT banned, read, accessible, write, upload, moderator, admin + FROM user_permissions + WHERE room = :r AND "user" = :u + """, + r=self.id, + u=user.id, + ).first() + self._perm_cache[user.id] = [bool(c) for c in row] + + ( + is_banned, + can_read, + can_access, + can_write, + can_upload, + is_mod, + is_admin, + ) = self._perm_cache[user.id] + + # Shortcuts for check_permission calls + def check_unbanned(self, room: Room, user: Optional[User]): + return self.check_permission_for(room, user) + + def check_read(self, room: Room, user: Optional[User] = None): + return self.check_permission_for(room, user, read=True) + + def check_accessible(self, room: Room, user: Optional[User] = None): + return self.check_permission_for(room, user, accessible=True) + + def check_write(self, room: Room, user: Optional[User] = None): + return self.check_permission_for(room, user, write=True) + + def check_upload(self, room: Room, user: Optional[User] = None): + """Checks for both upload *and* write permission""" + return self.check_permission_for(room, user, write=True, upload=True) + + def check_moderator(self, room: Room, user: Optional[User]): + return self.check_permission_for(room, user, moderator=True) + + def check_admin(self, room: Room, user: Optional[User]): + return self.check_permission_for(room, user, admin=True) + + def receive_message( + self, + room: Room, + user: User, + data: bytes, + sig: bytes, + *, + whisper_to: Optional[Union[User, str]] = None, + whisper_mods: bool = False, + files: List[int] = [], + ): + if not self.check_write(user): + raise BadPermission() + + if data is None or sig is None or len(sig) != 64: + raise InvalidData() + + whisper_mods = bool(whisper_mods) + if (whisper_to or whisper_mods) and not self.check_moderator(user): + app.logger.warning(f"Cannot post a whisper to {room}: {user} is not a moderator") + raise BadPermission() + + if whisper_to and not isinstance(whisper_to, User): + whisper_to = User(session_id=whisper_to, autovivify=True, touch=False) + + filtered = (self.filter)[False, self.filter.read_message(user, data, room)] + + with db.transaction(): + if room.rate_limit_size and not self.check_admin(user): + since_limit = time.time() - room.rate_limit_interval + recent_count = query( + """ + SELECT COUNT(*) FROM messages + WHERE room = :r AND "user" = :u AND posted >= :since + """, + r=self.id, + u=user.id, + since=since_limit, + ).first()[0] + + if recent_count >= room.rate_limit_size: + raise PostRateLimited() + + data_size = len(data) + unpadded_data = utils.remove_session_message_padding(data) + + msg_id = db.insert_and_get_pk( + """ + INSERT INTO messages + (room, "user", data, data_size, signature, filtered, whisper, whisper_mods) + VALUES + (:r, :u, :data, :data_size, :signature, :filtered, :whisper, :whisper_mods) + """, + "id", + r=room.id, + u=user.id, + data=unpadded_data, + data_size=data_size, + signature=sig, + filtered=filtered is not None, + whisper=whisper_to.id if whisper_to else None, + whisper_mods=whisper_mods, + ) + + if files: + # Take ownership of any uploaded files attached to the post: + room._own_files(msg_id, files, user) + + assert msg_id is not None + row = query("SELECT posted, seqno FROM messages WHERE id = :m", m=msg_id).first() + msg = { + 'id': msg_id, + 'session_id': user.session_id, + 'posted': row[0], + 'seqno': row[1], + 'data': data, + 'signature': sig, + 'reactions': {}, + } + if filtered is not None: + msg['filtered'] = True + if whisper_to or whisper_mods: + msg['whisper'] = True + msg['whisper_mods'] = whisper_mods + if whisper_to: + msg['whisper_to'] = whisper_to.session_id + + if room._bot_status(): + add_bot_logic = 3 + """ + TODO: add logic for bots receiving message and doing + bot things. The bots should be queried in terms of + priority, + """ + + omq_global.send_mule("message_posted", msg["id"]) + return msg diff --git a/sogs/model/filter.py b/sogs/model/filter.py new file mode 100644 index 00000000..60191587 --- /dev/null +++ b/sogs/model/filter.py @@ -0,0 +1,206 @@ +from __future__ import annotations +from .. import config, crypto, session_pb2 as protobuf +import random + +from .. import crypto, config +from ..db import query +from ..hashing import blake2b +from nacl.signing import SigningKey +from .exc import PostRejected +from sogs.model.user import User +from sogs.model.room import Room, alphabet_filter_patterns +from sogs.model.post import Post + +import time + + +class SimpleFilter: + """ + Class representing a simple word filter searching for naughty words + + Object Properties: + bot - bot this filter is servicing + current_message - reference to current data being analyzed + """ + + def __init__(self): + self.current_message: Post = None + + def filtering(self): + settings = { + 'profanity_filter': config.PROFANITY_FILTER, + 'profanity_silent': config.PROFANITY_SILENT, + 'alphabet_filters': config.ALPHABET_FILTERS, + 'alphabet_silent': config.ALPHABET_SILENT, + } + if self.token in config.ROOM_OVERRIDES: + for k in ( + 'profanity_filter', + 'profanity_silent', + 'alphabet_filters', + 'alphabet_silent', + ): + if k in config.ROOM_OVERRIDES[self.token]: + settings[k] = config.ROOM_OVERRIDES[self.token][k] + return settings + + def filter_should_reply(self, filter_type, filter_lang, room: Room): + """If the settings say we should reply to a filter, this returns a tuple of + + (profile name, message format, whisper) + + where profile name is the name we should use in the reply, message format is a string with + substitutions ready, and whisper is True/False depending on whether it should be whispered + to the user (True) or public (False). + + If we shouldn't reply this returns (None, None, None) + """ + + if not config.FILTER_SETTINGS: + return (None, None, None) + + reply_format = None + profile_name = 'SOGS' + public = False + + # Precedences from least to most specific so that we load values from least specific first + # then overwrite them if we find a value in a more specific section + room_precedence = ('*', room.token) + filter_precedence = ('*', filter_type, filter_lang) if filter_lang else ('*', filter_type) + + for r in room_precedence: + s1 = config.FILTER_SETTINGS.get(r) + if s1 is None: + continue + for f in filter_precedence: + settings = s1.get(f) + if settings is None: + continue + + rf = settings.get('reply') + pn = settings.get('profile_name') + pb = settings.get('public') + if rf is not None: + reply_format = random.choice(rf) + if pn is not None: + profile_name = pn + if pb is not None: + public = pb + + return (reply_format, profile_name, public) + + def read_message(self, user: User, data: bytes, room: Room): + """ + Checks a message for disallowed alphabets and profanity (if the profanity + filter is enabled). + + - Returns None if this message passes (i.e. didn't trigger any filter, or is + being posted by an admin to whom the filters don't apply). + + - Returns a callback if the message fails but should be silently accepted. The callback + should be called (with no arguments) *after* the filtered message is inserted into the db. + + - Throws PostRejected if the message should be rejected (and rejection passed back to the + user) + """ + + if not data: + raise ValueError('No message data passed to filter') + + self.current_message = Post(raw=data) + + if not config.FILTER_MODS and self.check_moderator(user): + return None + + filt = self.filtering() + alphabets = filt['alphabet_filters'] + for lang, pattern in alphabet_filter_patterns: + if lang not in alphabets: + continue + + if not pattern.search(self.current_message().text): + continue + + # Filter it! + filter_type, filter_lang = 'alphabet', lang + break + + if not filter_type and filt['profanity_filter']: + import better_profanity + + for part in (self.current_message().text, self.current_message().username): + if better_profanity.profanity.contains_profanity(part): + filter_type = 'profanity' + break + + if not filter_type: + return None + + silent = filt[filter_type + '_silent'] + + msg_fmt, prof_name, pub = self.filter_should_reply(filter_type, filter_lang) + if msg_fmt: + pbmsg = protobuf.Content() + body = msg_fmt.format( + profile_name=( + user.session_id + if self.current_message().username is None + else self.current_message().username + ), + profile_at="@" + user.session_id, + room_name=self.name, + room_token=self.token, + ).encode() + pbmsg.dataMessage.body = body + pbmsg.dataMessage.timestamp = int(time.time() * 1000) + pbmsg.dataMessage.profile.displayName = prof_name + + # Add two bytes padding so that session doesn't get confused by a lack of padding + pbmsg = pbmsg.SerializeToString() + b'\x80\x00' + + # Make a fake signing key based on prof_name and the server privkey (so that different + # names use different keys; otherwise the bot names overwrite each other in Session + # clients when a later message has a new profile name). + global filter_privkeys + if prof_name in room.filter_privkeys: + signingkey = filter_privkeys[prof_name] + else: + signingkey = SigningKey( + blake2b( + prof_name.encode() + crypto.server_signkey.encode(), key=b'sogsfiltering' + ) + ) + filter_privkeys[prof_name] = signingkey + + sig = signingkey.sign(pbmsg).signature + server_fake_user = User( + session_id='15' + signingkey.verify_key.encode().hex(), autovivify=True, touch=False + ) + + def insert_reply(): + query( + """ + INSERT INTO messages + (room, "user", data, data_size, signature, whisper) + VALUES + (:r, :u, :data, :data_size, :signature, :whisper) + """, + r=room.id, + u=server_fake_user.id, + data=pbmsg[:-2], + data_size=len(pbmsg), + signature=sig, + whisper=None if pub else user.id, + ) + + if filt[filter_type + '_silent']: + # Defer the insertion until after the filtered row gets inserted + return insert_reply + else: + insert_reply() + + elif silent: + return lambda: None + + # TODO: can we send back some error code that makes Session not retry? + raise PostRejected(f"filtration rejected message ({filter_type})") diff --git a/sogs/model/post.py b/sogs/model/post.py index b0d3a5a0..463ad1fc 100644 --- a/sogs/model/post.py +++ b/sogs/model/post.py @@ -18,17 +18,17 @@ def __init__(self, raw=None, *, user=None, text=None): @property def text(self): - """ accessor for the post body """ + """accessor for the post body""" return self._proto.body @property def username(self): - """ accessor for the username of the post's author """ + """accessor for the username of the post's author""" if self.profile is None: return return self.profile.displayName @property def profile(self): - """ accessor for the user profile data containing things like username etc """ + """accessor for the user profile data containing things like username etc""" return self._proto.profile diff --git a/sogs/model/room.py b/sogs/model/room.py index 7217b116..3132584d 100644 --- a/sogs/model/room.py +++ b/sogs/model/room.py @@ -1,7 +1,7 @@ from .. import config, crypto, db, utils, session_pb2 as protobuf from ..db import query from ..hashing import blake2b -from ..omq import send_mule +from ..omq import omq_global from ..web import app from .user import User from .file import File @@ -17,6 +17,8 @@ PostRateLimited, InvalidData, ) +from model.bot import Bot +from model.filter import SimpleFilter import os import random @@ -26,12 +28,6 @@ from typing import Optional, Union, List -# TODO: These really should be room properties, not random global constants (these -# are carried over from old SOGS). -rate_limit_size = 5 -rate_limit_interval = 16.0 - - # Character ranges for different filters. This is ordered because some are subsets of each other # (e.g. persian is a subset of the arabic character range). alphabet_filter_patterns = [ @@ -71,14 +67,25 @@ class Room: default_accessible - True if default user permissions include accessible permission default_write - True if default user permissions includes write permission default_upload - True if default user permissions includes file upload permission + + NEW: + default_bot - default_bot object reference + active_bot - true if default_bot moderator is actively moderating or false if passively filtering """ - def __init__(self, row=None, *, id=None, token=None): + def __init__(self, row=None, *, id=None, token=None, bot: Bot = None): """ Constructs a room from a pre-retrieved row *or* via lookup of a room token or id. When looking up this raises a NoSuchRoom if no room with that token/id exists. """ self._refresh(id=id, token=token, row=row) + self._active_bot: bool = False + self.rate_limit_size = 5 + self.rate_limit_interval = 16.0 + + @property.getter + def _bot_status(self) -> bool: + return self._active_bot def _refresh(self, *, id=None, token=None, row=None, perms=False): """ @@ -426,114 +433,6 @@ def active_users_last(self, cutoff: float): since=time.time() - cutoff, ).first()[0] - def check_permission( - self, - user: Optional[User] = None, - *, - admin=False, - moderator=False, - read=False, - accessible=False, - write=False, - upload=False, - ): - """ - Checks whether `user` has the required permissions for this room and isn't banned. Returns - True if the user satisfies the permissions, False otherwise. If no user is provided then - permissions are checked against the room's defaults. - - Looked up permissions are cached within the Room instance so that looking up the same user - multiple times (i.e. from multiple parts of the code) does not re-query the database. - - Named arguments are as follows: - - admin -- if true then the user must have admin access to the room - - moderator -- if true then the user must have moderator (or admin) access to the room - - read -- if true then the user must have read access - - accessible -- if true then the user must have accessible access; note that this permission - is satisfied by *either* the `accessible` or `read` database flags (that is: read implies - accessible). - - write -- if true then the user must have write access - - upload -- if true then the user must have upload access; this should usually be combined - with write=True. - - You can specify multiple permissions as True, in which case all must be satisfied. If you - specify no permissions as required then the check only checks whether a user is banned but - otherwise requires no specific permission. - """ - - if user is None: - is_banned, can_read, can_access, can_write, can_upload, is_mod, is_admin = ( - False, - bool(self.default_read), - bool(self.default_accessible), - bool(self.default_write), - bool(self.default_upload), - False, - False, - ) - else: - if user.id not in self._perm_cache: - row = query( - """ - SELECT banned, read, accessible, write, upload, moderator, admin - FROM user_permissions - WHERE room = :r AND "user" = :u - """, - r=self.id, - u=user.id, - ).first() - self._perm_cache[user.id] = [bool(c) for c in row] - - ( - is_banned, - can_read, - can_access, - can_write, - can_upload, - is_mod, - is_admin, - ) = self._perm_cache[user.id] - - if is_admin: - return True - if admin: - return False - if is_mod: - return True - if moderator: - return False - return ( - not is_banned - and (not accessible or can_access or can_read) - and (not read or can_read) - and (not write or can_write) - and (not upload or can_upload) - ) - - # Shortcuts for check_permission calls - - def check_unbanned(self, user: Optional[User]): - return self.check_permission(user) - - def check_read(self, user: Optional[User] = None): - return self.check_permission(user, read=True) - - def check_accessible(self, user: Optional[User] = None): - return self.check_permission(user, accessible=True) - - def check_write(self, user: Optional[User] = None): - return self.check_permission(user, write=True) - - def check_upload(self, user: Optional[User] = None): - """Checks for both upload *and* write permission""" - return self.check_permission(user, write=True, upload=True) - - def check_moderator(self, user: Optional[User]): - return self.check_permission(user, moderator=True) - - def check_admin(self, user: Optional[User]): - return self.check_permission(user, admin=True) - def messages_size(self): """Returns the number and total size (in bytes) of non-deleted messages currently stored in this room. Size is reflects the size of uploaded message bodies, not necessarily the size @@ -604,7 +503,7 @@ def get_messages_for( padding-trimmed value actually stored in the database). """ - mod = self.check_moderator(user) + mod = self.default_bot.check_moderator(self, user) msgs = [] opt_count = sum(arg is not None for arg in (sequence, after, before, single)) + bool(recent) @@ -735,190 +634,6 @@ def get_messages_for( return msgs - def filtering(self): - settings = { - 'profanity_filter': config.PROFANITY_FILTER, - 'profanity_silent': config.PROFANITY_SILENT, - 'alphabet_filters': config.ALPHABET_FILTERS, - 'alphabet_silent': config.ALPHABET_SILENT, - } - if self.token in config.ROOM_OVERRIDES: - for k in ( - 'profanity_filter', - 'profanity_silent', - 'alphabet_filters', - 'alphabet_silent', - ): - if k in config.ROOM_OVERRIDES[self.token]: - settings[k] = config.ROOM_OVERRIDES[self.token][k] - return settings - - def filter_should_reply(self, filter_type, filter_lang): - """If the settings say we should reply to a filter, this returns a tuple of - - (profile name, message format, whisper) - - where profile name is the name we should use in the reply, message format is a string with - substitutions ready, and whisper is True/False depending on whether it should be whispered - to the user (True) or public (False). - - If we shouldn't reply this returns (None, None, None) - """ - - if not config.FILTER_SETTINGS: - return (None, None, None) - - reply_format = None - profile_name = 'SOGS' - public = False - - # Precedences from least to most specific so that we load values from least specific first - # then overwrite them if we find a value in a more specific section - room_precedence = ('*', self.token) - filter_precedence = ('*', filter_type, filter_lang) if filter_lang else ('*', filter_type) - - for r in room_precedence: - s1 = config.FILTER_SETTINGS.get(r) - if s1 is None: - continue - for f in filter_precedence: - settings = s1.get(f) - if settings is None: - continue - - rf = settings.get('reply') - pn = settings.get('profile_name') - pb = settings.get('public') - if rf is not None: - reply_format = random.choice(rf) - if pn is not None: - profile_name = pn - if pb is not None: - public = pb - - return (reply_format, profile_name, public) - - def should_filter(self, user: User, data: bytes): - """ - Checks a message for disallowed alphabets and profanity (if the profanity - filter is enabled). - - - Returns None if this message passes (i.e. didn't trigger any filter, or is - being posted by an admin to whom the filters don't apply). - - - Returns a callback if the message fails but should be silently accepted. The callback - should be called (with no arguments) *after* the filtered message is inserted into the db. - - - Throws PostRejected if the message should be rejected (and rejection passed back to the - user). - """ - msg_ = None - - def msg(): - nonlocal msg_ - if msg_ is None: - msg_ = Post(raw=data) - return msg_ - - if not config.FILTER_MODS and self.check_moderator(user): - return None - - filter_type = None - filter_lang = None - - filt = self.filtering() - alphabets = filt['alphabet_filters'] - for lang, pattern in alphabet_filter_patterns: - if lang not in alphabets: - continue - - if not pattern.search(msg().text): - continue - - # Filter it! - filter_type, filter_lang = 'alphabet', lang - break - - if not filter_type and filt['profanity_filter']: - import better_profanity - - for part in (msg().text, msg().username): - if better_profanity.profanity.contains_profanity(part): - filter_type = 'profanity' - break - - if not filter_type: - return None - - silent = filt[filter_type + '_silent'] - - msg_fmt, prof_name, pub = self.filter_should_reply(filter_type, filter_lang) - if msg_fmt: - pbmsg = protobuf.Content() - body = msg_fmt.format( - profile_name=(user.session_id if msg().username is None else msg().username), - profile_at="@" + user.session_id, - room_name=self.name, - room_token=self.token, - ).encode() - pbmsg.dataMessage.body = body - pbmsg.dataMessage.timestamp = int(time.time() * 1000) - pbmsg.dataMessage.profile.displayName = prof_name - - # Add two bytes padding so that session doesn't get confused by a lack of padding - pbmsg = pbmsg.SerializeToString() + b'\x80\x00' - - # Make a fake signing key based on prof_name and the server privkey (so that different - # names use different keys; otherwise the bot names overwrite each other in Session - # clients when a later message has a new profile name). - global filter_privkeys - if prof_name in filter_privkeys: - signingkey = filter_privkeys[prof_name] - else: - signingkey = SigningKey( - blake2b( - prof_name.encode() + crypto.server_signkey.encode(), key=b'sogsfiltering' - ) - ) - filter_privkeys[prof_name] = signingkey - - sig = signingkey.sign(pbmsg).signature - server_fake_user = User( - session_id='15' + signingkey.verify_key.encode().hex(), autovivify=True, touch=False - ) - - def insert_reply(): - query( - """ - INSERT INTO messages - (room, "user", data, data_size, signature, whisper) - VALUES - (:r, :u, :data, :data_size, :signature, :whisper) - """, - r=self.id, - u=server_fake_user.id, - data=pbmsg[:-2], - data_size=len(pbmsg), - signature=sig, - whisper=None if pub else user.id, - ) - - if filt[filter_type + '_silent']: - # Defer the insertion until after the filtered row gets inserted - return insert_reply - else: - insert_reply() - - elif silent: - # No reply, so just return an empty callback - def noop(): - pass - - return noop - - # FIXME: can we send back some error code that makes Session not retry? - raise PostRejected(f"filtration rejected message ({filter_type})") - def _own_files(self, msg_id: int, files: List[int], user): """ Associated any of the given file ids with the given message id. Only files that are recent, @@ -949,110 +664,6 @@ def _own_files(self, msg_id: int, files: List[int], user): bind_expanding=['ids'], ) - def add_post( - self, - user: User, - data: bytes, - sig: bytes, - *, - whisper_to: Optional[Union[User, str]] = None, - whisper_mods: bool = False, - files: List[int] = [], - ): - """ - Adds a post to the room. The user must have write permissions. - - Raises BadPermission() if the user doesn't have posting permission; PostRejected() if the - post was rejected (such as subclass PostRateLimited() if the post was rejected for too - frequent posting). - - Returns the message details. - """ - if not self.check_write(user): - raise BadPermission() - - if data is None or sig is None or len(sig) != 64: - raise InvalidData() - - whisper_mods = bool(whisper_mods) - if (whisper_to or whisper_mods) and not self.check_moderator(user): - app.logger.warning(f"Cannot post a whisper to {self}: {user} is not a moderator") - raise BadPermission() - - if whisper_to and not isinstance(whisper_to, User): - whisper_to = User(session_id=whisper_to, autovivify=True, touch=False) - - filtered = self.should_filter(user, data) - - with db.transaction(): - if rate_limit_size and not self.check_admin(user): - since_limit = time.time() - rate_limit_interval - recent_count = query( - """ - SELECT COUNT(*) FROM messages - WHERE room = :r AND "user" = :u AND posted >= :since - """, - r=self.id, - u=user.id, - since=since_limit, - ).first()[0] - - if recent_count >= rate_limit_size: - raise PostRateLimited() - - data_size = len(data) - unpadded_data = utils.remove_session_message_padding(data) - - msg_id = db.insert_and_get_pk( - """ - INSERT INTO messages - (room, "user", data, data_size, signature, filtered, whisper, whisper_mods) - VALUES - (:r, :u, :data, :data_size, :signature, :filtered, :whisper, :whisper_mods) - """, - "id", - r=self.id, - u=user.id, - data=unpadded_data, - data_size=data_size, - signature=sig, - filtered=filtered is not None, - whisper=whisper_to.id if whisper_to else None, - whisper_mods=whisper_mods, - ) - - if files: - # Take ownership of any uploaded files attached to the post: - self._own_files(msg_id, files, user) - - assert msg_id is not None - row = query("SELECT posted, seqno FROM messages WHERE id = :m", m=msg_id).first() - msg = { - 'id': msg_id, - 'session_id': user.session_id, - 'posted': row[0], - 'seqno': row[1], - 'data': data, - 'signature': sig, - 'reactions': {}, - } - if filtered is not None: - msg['filtered'] = True - if whisper_to or whisper_mods: - msg['whisper'] = True - msg['whisper_mods'] = whisper_mods - if whisper_to: - msg['whisper_to'] = whisper_to.session_id - - # Don't call this inside the transaction because, if it's inserting a reply, we want the - # reply to have a later timestamp for proper ordering (because the timestamp inside a - # transaction is the time when the transaction started). - if filtered is not None: - filtered() - - send_mule("message_posted", msg['id']) - return msg - def edit_post(self, user: User, msg_id: int, data: bytes, sig: bytes, *, files: List[int] = []): """ Edits a post in the room. The post must exist, must have been authored by the same user, @@ -1068,7 +679,7 @@ def edit_post(self, user: User, msg_id: int, data: bytes, sig: bytes, *, files: profanity filter. - NoSuchPost() if the post is deleted. """ - if not self.check_write(user): + if not self.default_bot.check_write(self, user): raise BadPermission() if data is None or sig is None or len(sig) != 64: @@ -1114,7 +725,7 @@ def edit_post(self, user: User, msg_id: int, data: bytes, sig: bytes, *, files: # If the edit includes new attachments then own them: self._own_files(msg_id, files, user) - send_mule("message_edited", msg_id) + omq_global.send_mule("message_edited", msg_id) def delete_posts(self, message_ids: List[int], deleter: User): """ @@ -1151,7 +762,7 @@ def delete_posts(self, message_ids: List[int], deleter: User): ) if ids: - if not self.check_moderator(deleter): + if not self.default_bot.check_moderator(self, deleter): # If not a moderator then we only proceed if all of the messages are the # user's own: res = query( @@ -1186,9 +797,11 @@ def delete_all_posts(self, poster: User, *, deleter: User): """ fail = None - if poster.id != deleter.id and not self.check_moderator(deleter): + if poster.id != deleter.id and not self.default_bot.check_moderator(self, deleter): fail = "user is not a moderator" - elif self.check_admin(poster) and not self.check_admin(deleter): + elif self.default_bot.check_admin(self, poster) and not self.default_bot.check_admin( + self, deleter + ): fail = "only admins can delete all posts of another admin" if fail is not None: @@ -1360,7 +973,11 @@ def _check_reaction_request( if user_required and not user: app.logger.warning("Reaction request requires user authentication") raise BadPermission() - if not (self.check_moderator(user) if mod_required else self.check_read(user)): + if not ( + self.default_bot.check_moderator(self, user) + if mod_required + else self.default_bot.check_read(self, user) + ): app.logger.warning("Reaction request requires moderator authentication") raise BadPermission() @@ -1515,7 +1132,7 @@ def get_mods(self, user=None): ([public_mods], [public_admins], [hidden_mods], [hidden_admins]) """ - visible_clause = "" if self.check_moderator(user) else "AND visible_mod" + visible_clause = "" if self.default_bot.check_moderator(self, user) else "AND visible_mod" m, hm, a, ha = [], [], [], [] for session_id, visible, admin in query( f""" @@ -1570,7 +1187,7 @@ def set_moderator(self, user: User, *, added_by: User, admin=False, visible=True added_by is the user performing the update and must have admin permission. """ - if not self.check_admin(added_by): + if not self.default_bot.check_admin(self, added_by): app.logger.warning( f"Unable to set {user} as {'admin' if admin else 'moderator'} of {self}: " f"{added_by} is not an admin" @@ -1615,7 +1232,7 @@ def remove_moderator(self, user: User, *, removed_by: User, remove_admin_only: b a room moderator if already a room moderator or admin. """ - if not self.check_admin(removed_by): + if not self.default_bot.check_admin(self, removed_by): raise BadPermission() with db.transaction(): @@ -1654,13 +1271,15 @@ def ban_user(self, to_ban: User, *, mod: User, timeout: Optional[float] = None): with db.transaction(): with to_ban.check_blinding() as to_ban: fail = None - if not self.check_moderator(mod): + if not self.default_bot.check_moderator(self, mod): fail = "user is not a moderator" elif to_ban.id == mod.id: fail = "self-ban not permitted" elif to_ban.global_moderator: fail = "global mods/admins cannot be banned" - elif self.check_moderator(to_ban) and not self.check_admin(mod): + elif self.default_bot.check_moderator( + self, to_ban + ) and not self.default_bot.check_admin(self, mod): fail = "only admins can ban room mods/admins" if fail is not None: @@ -1714,7 +1333,7 @@ def unban_user(self, to_unban: User, *, mod: User): Throws on other errors (e.g. permission denied). """ - if not self.check_moderator(mod): + if not self.default_bot.check_moderator(self, mod): app.logger.warning(f"Error unbanning {to_unban} from {self} by {mod}: not a moderator") raise BadPermission() @@ -1785,7 +1404,7 @@ def set_permissions(self, user: User, *, mod: User, **perms): "Room.set_permissions: at least one of {', '.join(perm_types)} must be specified" ) - if not self.check_moderator(mod): + if not self.default_bot.check_moderator(self, mod): app.logger.warning(f"Error set perms {perms} on {user} by {mod}: not a moderator") raise BadPermission() @@ -1870,7 +1489,7 @@ def clear_future_permissions( def add_future_permission( self, - user, + user: User, *, at: float, mod: User, @@ -1945,7 +1564,7 @@ def upload_file( Returns the id of the newly inserted file row. Throws on error. """ - if not self.check_upload(uploader): + if not self.default_bot.check_upload(self, uploader): raise BadPermission() files_dir = os.path.join(config.UPLOAD_PATH, self.token) @@ -2037,7 +1656,7 @@ def pin(self, msg_id: int, admin: User): reorder pins, which are always sorted oldest-to-newest). """ - if not self.check_admin(admin): + if not self.default_bot.check_admin(self, admin): app.logger.warning(f"Unable to pin message to {self}: {admin} is not an admin") raise BadPermission() @@ -2064,7 +1683,7 @@ def unpin_all(self, admin: User): number of pinned messages removed. """ - if not self.check_admin(admin): + if not self.default_bot.check_admin(self, admin): app.logger.warning("Unable to unpin all messages from {self}: {admin} is not an admin") raise BadPermission() @@ -2096,7 +1715,7 @@ def unpin(self, msg_id: int, admin: User): pinned messages actually removed (i.e. 0 or 1). """ - if not self.check_admin(admin): + if not self.default_bot.check_admin(self, admin): app.logger.warning("Unable to unpin message from {self}: {admin} is not an admin") raise BadPermission() diff --git a/sogs/mule.py b/sogs/mule.py index f6777143..654243a2 100644 --- a/sogs/mule.py +++ b/sogs/mule.py @@ -8,72 +8,15 @@ from .web import app from . import cleanup from . import config -from . import omq as o +from .omq import OMQ +from .model.clientmanager import ClientManager + # This is the uwsgi "mule" that handles things not related to serving HTTP requests: # - it holds the oxenmq instance (with its own interface into sogs) # - it handles cleanup jobs (e.g. periodic deletions) -def run(): - try: - app.logger.info("OxenMQ mule started.") - - while True: - time.sleep(1) - - except Exception: - app.logger.error("mule died via exception:\n{}".format(traceback.format_exc())) - - -def allow_conn(addr, pk, sn): - # TODO: user recognition auth - return oxenmq.AuthLevel.basic - - -def admin_conn(addr, pk, sn): - return oxenmq.AuthLevel.admin - - -def inproc_fail(connid, reason): - raise RuntimeError(f"Couldn't connect mule to itself: {reason}") - - -def setup_omq(): - omq = o.omq - - app.logger.debug("Mule setting up omq") - if isinstance(config.OMQ_LISTEN, list): - listen = config.OMQ_LISTEN - elif config.OMQ_LISTEN is None: - listen = [] - else: - listen = [config.OMQ_LISTEN] - for addr in listen: - omq.listen(addr, curve=True, allow_connection=allow_conn) - app.logger.info(f"OxenMQ listening on {addr}") - - # Internal socket for workers to talk to us: - omq.listen(config.OMQ_INTERNAL, curve=False, allow_connection=admin_conn) - - # Periodic database cleanup timer: - omq.add_timer(cleanup.cleanup, timedelta(seconds=cleanup.INTERVAL)) - - # Commands other workers can send to us, e.g. for notifications of activity for us to know about - worker = omq.add_category("worker", access_level=oxenmq.AuthLevel.admin) - worker.add_command("message_posted", message_posted) - worker.add_command("messages_deleted", messages_deleted) - worker.add_command("message_edited", message_edited) - - app.logger.debug("Mule starting omq") - omq.start() - - # Connect mule to itself so that if something the mule does wants to send something to the mule - # it will work. (And so be careful not to recurse!) - app.logger.debug("Mule connecting to self") - o.mule_conn = omq.connect_inproc(on_success=None, on_failure=inproc_fail) - - def log_exceptions(f): @functools.wraps(f) def wrapper(*args, **kwargs): @@ -86,18 +29,83 @@ def wrapper(*args, **kwargs): return wrapper -@log_exceptions -def message_posted(m: oxenmq.Message): - id = bt_deserialize(m.data()[0]) - app.logger.debug(f"FIXME: mule -- message posted stub, id={id}") - - -@log_exceptions -def messages_deleted(m: oxenmq.Message): - ids = bt_deserialize(m.data()[0]) - app.logger.debug(f"FIXME: mule -- message delete stub, deleted messages: {ids}") +class Mule: + def __init__(self): + self.run() - -@log_exceptions -def message_edited(m: oxenmq.Message): - app.logger.debug("FIXME: mule -- message edited stub") + def run(self): + try: + app.logger.info("OxenMQ mule started.") + + while True: + time.sleep(1) + + except Exception: + app.logger.error("mule died via exception:\n{}".format(traceback.format_exc())) + + def allow_conn(self, addr, pk, sn): + # TODO: user recognition auth + return oxenmq.AuthLevel.basic + + def admin_conn(self, addr, pk, sn): + return oxenmq.AuthLevel.admin + + def inproc_fail(self, connid, reason): + raise RuntimeError(f"Couldn't connect mule to itself: {reason}") + + @log_exceptions + def message_posted(self, m: oxenmq.Message): + id = bt_deserialize(m.data()[0]) + app.logger.debug(f"FIXME: mule -- message posted stub, id={id}") + + @log_exceptions + def messages_deleted(self, m: oxenmq.Message): + ids = bt_deserialize(m.data()[0]) + app.logger.debug(f"FIXME: mule -- message delete stub, deleted messages: {ids}") + + @log_exceptions + def message_edited(self, m: oxenmq.Message): + app.logger.debug("FIXME: mule -- message edited stub") + + def setup_omq(self, omq: OMQ): + app.logger.debug("Mule setting up omq") + if isinstance(config.OMQ_LISTEN, list): + listen = config.OMQ_LISTEN + elif config.OMQ_LISTEN is None: + listen = [] + else: + listen = [config.OMQ_LISTEN] + for addr in listen: + omq.listen(addr, curve=True, allow_connection=self.allow_conn) + app.logger.info(f"OxenMQ listening on {addr}") + + # Internal socket for workers to talk to us: + omq._omq.listen(config.OMQ_INTERNAL, curve=False, allow_connection=self.admin_conn) + + # Periodic database cleanup timer: + self._omq.add_timer(cleanup.cleanup, timedelta(seconds=cleanup.INTERVAL)) + + # Commands other workers can send to us, e.g. for notifications of activity for us to know about + worker = omq._omq.add_category("worker", access_level=oxenmq.AuthLevel.admin) + worker.add_command("message_posted", self.message_posted) + worker.add_command("messages_deleted", self.messages_deleted) + worker.add_command("message_edited", self.message_edited) + + # client code + handler = omq._omq.add_category("handler", access_level=oxenmq.AuthLevel.admin) + handler.add_request_command("register_client", omq.register_client) + handler.add_request_command("deregister_client", omq.deregister_client) + handler.add_request_command("send_to_handler", omq.manager.receive_message) + + # proxy handler for subrequest queue + internal = omq._omq.add_category("internal", access_level=oxenmq.AuthLevel.admin) + internal.add_request_command("get_next_request", omq.get_next_request) + internal.add_request_command("subreq_response", omq.subreq_response) + + app.logger.debug("Mule starting omq") + self._omq.start() + + # Connect mule to itself so that if something the mule does wants to send something to the mule + # it will work. (And so be careful not to recurse!) + app.logger.debug("Mule connecting to self") + omq.mule_conn = omq.connect_inproc(on_success=None, on_failure=self.inproc_fail) diff --git a/sogs/omq.py b/sogs/omq.py index 06bf2b7b..4460ff2f 100644 --- a/sogs/omq.py +++ b/sogs/omq.py @@ -1,67 +1,104 @@ # Common oxenmq object; this is used by workers and the oxenmq mule. We create, but do not start, # this pre-forking. -import oxenmq -from oxenc import bt_serialize +import oxenmq, queue +from oxenc import bt_serialize, bt_deserialize +from .mule import log_exceptions +from .routes import omq_auth from . import crypto, config from .postfork import postfork +from .model.clientmanager import ClientManager -omq = None -mule_conn = None -test_suite = False +omq_global = None -def make_omq(): - omq = oxenmq.OxenMQ(privkey=crypto._privkey.encode(), pubkey=crypto.server_pubkey.encode()) - # We have multiple workers talking to the mule, so we *must* use ephemeral ids to not replace - # each others' connections. - omq.ephemeral_routing_id = True +class OMQ: + @postfork + def __init__(self): + try: + import uwsgi + except ModuleNotFoundError: + return - return omq + self._omq = oxenmq.OxenMQ( + privkey=crypto._privkey.encode(), pubkey=crypto.server_pubkey.encode() + ) + self._omq.ephemeral_routing_id = True + self.client_map = {} + self.manager = ClientManager() + self.test_suite = False + self.subreq_queue = queue.SimpleQueue() + if uwsgi.mule_id() != 0: + uwsgi.opt['mule'].setup_omq(self) + return -# Postfork for workers: we start oxenmq and connect to the mule process -@postfork -def start_oxenmq(): - try: - import uwsgi - except ModuleNotFoundError: - return + uwsgi.register_signal(123, 'worker', self.handle_proxied_omq_req) - global omq, mule_conn + from .web import app # Imported here to avoid circular import - omq = make_omq() + app.logger.debug(f"Starting oxenmq connection to mule in worker {uwsgi.worker_id()}...") + self._omq.start() - if uwsgi.mule_id() != 0: - from . import mule + app.logger.debug("Started, connecting to mule...") + self.mule_conn = self._omq.connect_remote(oxenmq.Address(config.OMQ_INTERNAL)) - mule.setup_omq() - return + app.logger.debug(f"OMQ worker {uwsgi.worker_id()} connected to mule") - from .web import app # Imported here to avoid circular import + global omq_global + omq_global = self - app.logger.debug(f"Starting oxenmq connection to mule in worker {uwsgi.worker_id()}") + @log_exceptions + def subreq_response(self, msg: oxenmq.Message): + req_id, code, headers, data = bt_deserialize(msg.data()[0]) - omq.start() - app.logger.debug("Started, connecting to mule") - mule_conn = omq.connect_remote(oxenmq.Address(config.OMQ_INTERNAL)) + @log_exceptions + def handle_proxied_omq_req(self): + req_id, subreq_body = self.send_mule(command='get_next_request', prefix='internal') - app.logger.debug(f"worker {uwsgi.worker_id()} connected to mule OMQ") + # pass subrequest to omq endpoint + response, code = omq_auth.endpoint( + subreq_body['query'], subreq_body['pubkey'], subreq_body['params'] + ) + self.send_mule('subreq_response', req_id, code, response.headers, response.data) -def send_mule(command, *args, prefix="worker."): - """ - Sends a command to the mule from a worker (or possibly from the mule itself). The command will - be prefixed with "worker." (unless overridden). + @log_exceptions + def get_next_request(self): + try: + subreq_body = self.subreq_queue.get() + except: + raise RuntimeError('No subrequest found in queue') + req_id = list(subreq_body.keys())[0] + return req_id, subreq_body[id] - Any args will be bt-serialized and send as message parts. - """ - if prefix: - command = prefix + command + @log_exceptions + def register_client(self, msg: oxenmq.Message): + cid, authlevel, bot, priority = bt_deserialize(msg.data()[0]) + conn_id = msg.conn() + self.client_map[conn_id] = cid + self.manager.register_client(conn_id, cid, authlevel, bot, priority) - if test_suite and omq is None: - pass # TODO: for mule call testing we may want to do something else here? - else: - omq.send(mule_conn, command, *(bt_serialize(data) for data in args)) + @log_exceptions + def deregister_client(self, msg: oxenmq.Message): + cid, bot = bt_deserialize(msg.data()[0]) + self.client_map.pop(cid) + self.manager.deregister_client(cid, bot) + + def send_mule(self, command, *args, prefix="worker."): + """ + Sends a command to the mule from a worker (or possibly from the mule itself). The command will + be prefixed with "worker." (unless overridden). + + Any args will be bt-serialized and send as message parts. + """ + + if prefix: + command = prefix + command + + if self.test_suite and self._omq is None: + pass # TODO: for mule call testing we may want to do something else here? + else: + self._omq.send(self.mule_conn, command, *(bt_serialize(data) for data in args)) diff --git a/sogs/postfork.py b/sogs/postfork.py index 7c544c9d..6800e24b 100644 --- a/sogs/postfork.py +++ b/sogs/postfork.py @@ -11,7 +11,6 @@ def __init__(self, f): def __call__(self, f): pass - else: import uwsgidecorators diff --git a/sogs/routes/clients.py b/sogs/routes/clients.py new file mode 100644 index 00000000..12ca21d5 --- /dev/null +++ b/sogs/routes/clients.py @@ -0,0 +1,150 @@ +from .. import db, http, utils +from ..model import room as mroom +from ..model.user import User +from ..web import app +from ..omq import omq_global +from . import omq_auth + +from flask import abort, jsonify, g, Blueprint, request + +# User-related routes + +""" + TOFIX: + remove HTTP shit from requests + +""" + +clients = Blueprint('clients', __name__) +app.register_blueprint(clients) + + +@omq_auth.first_request +def register(cid): + """ + Registers a client with SOGS OMQ instance. In this context, "client" refers to any entity + seeking to create an authenticated OMQ connection. This may be, but is not limited to, + a user or a bot + + ## URL Parameters + + - 'cid': the client ID (session ID) of the given client to be registered with the SOGS instance + + ## Query Parameters + + - 'bot' (bool) : is bot or not + + ## Body Parameters + + Takes a JSON object as body with the following keys: + + - 'authlevel' : the oxenmq Authlevel to be attributed to the given client + - 'priority' : the priority level to be assigned to the given bot. If not passed, will be assigned + and handled by bot priority-queue + """ + + req = request.json + bot = utils.get_int_param('bot') # will set bot == 1 if key "bot" has value True + authlevel = req.get('authlevel') + priority = req.get('priority') + + client = (bot is 1)[register_client(cid, authlevel), register_bot(cid, authlevel, priority)] + + return client + + +@clients.post("/client/registered/bot/") +def register_bot(cid, authlevel, priority): + """ + Registers a bot with SOGS OMQ instance + + ## URL Parameters + + - 'cid': the client ID (session ID) of the given client to be registered with the SOGS instance + + ## Body Parameters + + Takes a JSON object as body with the following keys (passed as parameters from register()): + + - 'authlevel' : the oxenmq Authlevel to be attributed to the given client + - 'priority' : the priority level to be assigned to the given bot. If not passed, will be assigned + and handled by bot priority-queue + """ + + client = omq_global.send_mule( + command='register_client', + cid=cid, + authlevel=authlevel, + bot=1, + priority=priority, + prefix='handler', + ) + + return client + + +@clients.post("/client/registered/client/") +def register_client(cid, authlevel): + """ + Registers a non-bot client with SOGS OMQ instance + + ## URL Parameters + + - 'cid': the client ID (session ID) of the given client to be registered with the SOGS instance + + ## Body Parameters + + Takes a JSON object as body with the following keys (passed as parameters from register()): + + - 'authlevel' : the oxenmq Authlevel to be attributed to the given client + """ + + client = omq_global.send_mule( + command='register_client', + cid=cid, + authlevel=authlevel, + bot=0, + priority=None, + prefix='handler', + ) + + return client + + +@omq_auth.admin_required +def unregister(cid): + """ + Unegisters a non-bot client with SOGS OMQ instance + + ## URL Parameters + + - 'cid': the client ID (session ID) of the given client to be registered with the SOGS instance + + ## Query Parameters + + - 'bot' (bool) : is bot or not + """ + + bot = utils.get_int_param('bot') + + client = (bot)[unregister_client(cid), unregister_bot(cid)] + + return client + + +@clients.post("/client/deregistered/client/") +@clients.delete("/client/registered/client/") +def unregister_client(cid): + + client = omq_global.send_mule(command='unregister_client', cid=cid, bot=0, prefix='handler') + + return client + + +@clients.post("/bot/deregistered/bot/") +@clients.delete("/bot/registered/bot/") +def unregister_bot(cid): + + client = omq_global.send_mule(command='unregister_bot', cid=cid, bot=1, prefix='handler') + + return client diff --git a/sogs/routes/legacy.py b/sogs/routes/legacy.py index 1dee43b7..7eed1d30 100644 --- a/sogs/routes/legacy.py +++ b/sogs/routes/legacy.py @@ -2,7 +2,6 @@ from werkzeug.exceptions import HTTPException from ..web import app from .. import crypto, config, db, http, utils -from ..omq import send_mule from ..utils import jsonify_with_base64 from ..model.room import Room, get_accessible_rooms, get_deletions_deprecated from ..model.user import User @@ -186,7 +185,6 @@ def legacy_transform_message(m): @legacy.post("/messages") def handle_post_legacy_message(): - user, room = legacy_check_user_room(write=True) req = request.json @@ -322,7 +320,13 @@ def handle_legacy_delete_messages(ids=None): ids = room.delete_posts(ids, user) if ids: - send_mule("messages_deleted", ids) + # avoid circular imports + try: + from ..omq import omq_global + except ModuleNotFoundError: + return + + omq_global.send_mule("messages_deleted", ids) return jsonify({'status_code': http.OK}) diff --git a/sogs/routes/messages.py b/sogs/routes/messages.py index 34b65447..75f532e8 100644 --- a/sogs/routes/messages.py +++ b/sogs/routes/messages.py @@ -1,12 +1,15 @@ from .. import http, utils -from . import auth +from . import auth, omq_auth +from model.room import Room +from ..omq import omq_global +from ..web import app from flask import abort, jsonify, g, Blueprint, request # Room message retrieving/submitting endpoints - messages = Blueprint('messages', __name__) +app.register_blueprint(messages) def qs_reactors(): @@ -15,7 +18,7 @@ def qs_reactors(): @messages.get("/room//messages/since/") @auth.read_required -def messages_since(room, seqno): +def messages_since(room: Room, seqno): """ Retrieves message *updates* from a room. This is the main message polling endpoint in SOGS. @@ -99,7 +102,7 @@ def messages_since(room, seqno): @messages.get("/room//messages/before/") @auth.read_required -def messages_before(room, msg_id): +def messages_before(room: Room, msg_id): """ Retrieves messages from the room preceding a given id. @@ -142,7 +145,7 @@ def messages_before(room, msg_id): @messages.get("/room//messages/recent") @auth.read_required -def messages_recent(room): +def messages_recent(room: Room): """ Retrieves recent messages posted to this room. @@ -181,7 +184,7 @@ def messages_recent(room): @messages.get("/room//message/") @auth.read_required -def message_single(room, msg_id): +def message_single(room: Room, msg_id): """ Returns a single message by ID. @@ -308,9 +311,15 @@ def message_single(room, msg_id): return utils.jsonify_with_base64(msgs[0]) +""" + TOFIX: + - add some decorator to this s.t. it routes it to the correct OMQ endpoint +""" + + @messages.post("/room//message") @auth.user_required -def post_message(room): +def post_message(room: Room): """ Posts a new message to a room. @@ -359,8 +368,9 @@ def post_message(room): """ req = request.json - msg = room.add_post( - g.user, + msg = omq_global.manager.receive_message( + user=g.user, + room=room, data=utils.decode_base64(req.get('data')), sig=utils.decode_base64(req.get('signature')), whisper_to=req.get('whisper_to'), @@ -373,7 +383,7 @@ def post_message(room): @messages.put("/room//message/") @auth.user_required -def edit_message(room, msg_id): +def edit_message(room: Room, msg_id): """ Edits a message, replacing its existing content with new content and a new signature. @@ -420,7 +430,7 @@ def edit_message(room, msg_id): @messages.delete("/room//message/") @auth.user_required -def remove_message(room, msg_id): +def remove_message(room: Room, msg_id): """ Remove a message by its message id @@ -447,7 +457,7 @@ def remove_message(room, msg_id): @messages.post("/room//pin/") -def message_pin(room, msg_id): +def message_pin(room: Room, msg_id): """ Adds a pinned message to this room. @@ -490,7 +500,7 @@ def message_pin(room, msg_id): @messages.post("/room//unpin/") -def message_unpin(room, msg_id): +def message_unpin(room: Room, msg_id): """ Remove a message from this room's pinned message list. @@ -524,7 +534,7 @@ def message_unpin(room, msg_id): @messages.post("/room//unpin/all") -def message_unpin_all(room): +def message_unpin_all(room: Room): """ Removes *all* pinned messages from this room. @@ -553,7 +563,7 @@ def message_unpin_all(room): @messages.put("/room//reaction//") @auth.user_required @auth.read_required -def message_react(room, msg_id, reaction): +def message_react(room: Room, msg_id, reaction): """ Adds a reaction to the given message in this room. The user must have read access in the room. @@ -606,7 +616,7 @@ def message_react(room, msg_id, reaction): @messages.delete("/room//reaction//") @auth.user_required @auth.read_required -def message_unreact(room, msg_id, reaction): +def message_unreact(room: Room, msg_id, reaction): """ Removes a reaction from a post this room. The user must have read access in the room. This only removes the user's own reaction but does not affect the reactions of other users. @@ -642,7 +652,7 @@ def message_unreact(room, msg_id, reaction): @messages.delete("/room//reactions//") @messages.delete("/room//reactions/") @auth.mod_required -def message_delete_reactions(room, msg_id, reaction=None): +def message_delete_reactions(room: Room, msg_id, reaction=None): """ Removes all reactions of all users from a post in this room. The calling must have moderator permissions in the room. This endpoint can either remove a single reaction (e.g. remove all 🍆 @@ -677,7 +687,7 @@ def message_delete_reactions(room, msg_id, reaction=None): @messages.get("/room//reactors//") @auth.read_required -def message_get_reactors(room, msg_id, reaction): +def message_get_reactors(room: Room, msg_id, reaction): """ Returns the list of all reactors who have added a particular reaction to a particular message. diff --git a/sogs/routes/omq_auth.py b/sogs/routes/omq_auth.py new file mode 100644 index 00000000..870dd47c --- /dev/null +++ b/sogs/routes/omq_auth.py @@ -0,0 +1,169 @@ +import oxenmq, importlib +from . import auth +from ..web import app +from ..db import query +from .. import http +from ..omq import omq_global + +from typing import Callable +from flask import request, abort, Response, g, Blueprint +from functools import wraps + +# Authentication for handling OMQ requests + + +def endpoint(query, pubkey, params): + """ + Default endpoint for omq requests to pass sub-requests to; passthrough for flask HTTP request + + Message (request) components: + + "query" - the request query + "pubkey" - pk of client making request + "params" - a json value to dump as the the query parameters + + Example: + full request: '@omq.endpoint('room.messages_since', {'Room:room', 'int:seqno'})' + query: 'room.messages_since' + params: {'Room:room', 'int:seqno'} + """ + + assert ( + len(query.split('.')) == 2, + 'Error, query must be callable in format .', + ) + func = importlib.import_module(query) + + # unpack dictionary as request parameters + response, code = func(**params) + + return response, code + + +def abort_request(code, msg, warn=True): + if warn: + app.logger.warning(msg) + else: + app.logger.debug(msg) + abort(Response(msg, status=code, mimetype='text/plain')) + + +def require_client(): + """ + Requires that an authenticated client was found in the OMQ instance; aborts with + UNAUTHORIZED if the request has no client + """ + if g.client_id is None: + abort_request(http.UNAUTHORIZED, 'OMQ client authentication required') + + +def client_required(f): + """ + Decorator for an endpoint that requires a client; this calls require_client() at the + beginning of the request to abort the request as UNAUTHORIZED if the client has not been + previously authenticated + """ + + @wraps(f) + def required_client_wrapper(*args, **kwargs): + require_client() + return f(*args, **kwargs) + + return required_client_wrapper + + +def require_authlevel(admin=True): + require_client() + if ( + g.client_authlevel is not oxenmq.Authlevel.admin + if admin + else g.client_authlevel is not oxenmq.Authlevel.basic + ): + abort_request( + http.FORBIDDEN, + f"This endpoint requires oxenmq.Authlevel.{'admin' if admin else 'basic'} permissions", + ) + + +def basic_required(f): + """Decorator for an endpoint that requires a client has basic OMQ authorization""" + + @wraps(f) + def required_basic_wrapper(*args, **kwargs): + require_authlevel(admin=False) + return f(*args, **kwargs) + + return required_basic_wrapper + + +def admin_required(f): + """Decorator for an endpoint that requires a client has admin OMQ authorization""" + + @wraps(f) + def required_admin_wrapper(*args, **kwargs): + require_authlevel(admin=True) + return f(*args, **kwargs) + + return required_admin_wrapper + + +def first_request(f): + """Decorator for an endpoint that will be the very first request for a given client. This + will ensure that the client is then registered for any subsequent requests. + + This function will typically take the folling parameters: + - cid : unique client ID to be attributed + - authlevel (oxenmq) + """ + + @wraps + def first_request_wrapper(*args, cid, authlevel, **kwargs): + handle_omq_registration(cid, authlevel) + return f(*args, cid=cid, authlevel=authlevel, **kwargs) + + return first_request_wrapper + + +def handle_omq_registration(sid, authlevel): + """ + Registers client with OMQ instance before its very first request + """ + if hasattr(g, 'client_id') and hasattr(g, 'client_authlevel') and not g.client_reauth: + app.logger.warning( + f"Client {g.client_id} already registered for {g.client_authlevel} access" + ) + return + + """ + Here goes ye olde OMQ registration logic. We need to decide what identification will + be used to verify every connected client s.t. that information persists for all subsequent + requests + + In this registration, we need to set: + g.client_id + g.client_authlevel + """ + + +@app.before_request +def verify_omq_auth(): + """ + Verifies OMQ authentication before each request + """ + + # If there is already a g.o_id, then this is NOT the first request made by this client, unless + # g.client_reauth has been specifically set + if hasattr(g, 'client_id') and hasattr(g, 'client_authlevel') and not g.client_reauth: + app.logger.debug( + f"Client {g.client_id} already authenticated for {g.client_authlevel} access" + ) + return + + +""" + TOFIX: + - add some type of dict in omq_global to map conn_ID (onenmq conn ID) to session_ID/other info + - do not persist: + - room specific access: check every time it makes a request because it can change + - values that admin level can change +""" diff --git a/sogs/routes/omq_subrequest.py b/sogs/routes/omq_subrequest.py new file mode 100644 index 00000000..10488d02 --- /dev/null +++ b/sogs/routes/omq_subrequest.py @@ -0,0 +1,67 @@ +from ..web import app +from ..omq import omq_global + +from flask import request, g +from io import BytesIO +from typing import Optional, Union +import traceback, json, urllib.parse, oxenmq + + +def make_omq_subreq( + subreq_id, + endpoint: str, + query: str, + pubkey, + msg_later: oxenmq.Message.send_later, + params: Optional[Union[dict, list]] = None, + client_reauth: bool = False, +): + """ + Makes an omq subrequest from the given parameters, returns the response object and a dict of + lower-case response headers keys to header values + + Parameters: + subreq_id - randomly generated ID for subrequest + endpoint - the flask blueprint/endpoint to be queried + query - the callable module method in format . + pubkey - pk of client making request + msg_later - &oxenmq::Message::DeferredSend reference to be stored in subreq_queue along with data + params - a json value to dump as the the query parameters + """ + + if params is not None: + body = json.dumps(params, separators=(',', ':')).encode() + else: + body = b'' + + body_input = BytesIO(body) + content_length = len(body) + + subreq_body = { + subreq_id: { + 'endpoint': endpoint, + 'query': query, + 'pubkey': pubkey, + 'msg_later': msg_later, + 'params': params, + } + } + + try: + app.logger.debug(f"Injecting sub-request for omq.{endpoint} {query}") + g.client_reauth = client_reauth + + omq_global.subreq_queue.put(subreq_body) + + try: + import uwsgi + except ModuleNotFoundError: + return + + uwsgi.signal(123) + + except Exception: + app.logger.warning( + f"Sub-request for omq.{endpoint} {query} failed: {traceback.format_exc()}" + ) + raise diff --git a/sogs/routes/onion_omq_request.py b/sogs/routes/onion_omq_request.py new file mode 100644 index 00000000..dc070dff --- /dev/null +++ b/sogs/routes/onion_omq_request.py @@ -0,0 +1,107 @@ +from flask import request, abort, Blueprint +from nacl.utils import random +import oxenmq +import json + +from ..web import app +from .. import crypto, http, utils + +from .omq_subrequest import make_omq_subreq + + +def handle_v4onion_omqreq_plaintext(msg: oxenmq.Message): + """ + Handles a decrypted v4 onion request; this injects a subrequest to process it then returns the + result of that subrequest. In contrast to v3, it is more efficient (particularly for binary + input or output) and allows using endpoints that return headers or bodies with non-2xx response + codes. + + Message (request) components: + + "endpoint" - the omq endpoint + "query" - the request query + "pubkey" - pk of client making request + "params" - a json value to dump as the the query parameters + + Example: + full request: `omq.endpoint('some_endpoint', 'room.messages_since', jvi0grsj3029fjwhatever, {'Room:room', 'int:seqno'})` + endpoint: 'some_endpoint' + query: 'room.messages_since' + pubkey: jvi0grsj3029fjwhatever + params: {'Room:room', 'int:seqno'} + """ + + try: + body = msg.data()[0] + + if not (body.startswith(b'l') and body.endswith(b'e')): + raise RuntimeError("Invalid onion request body: expected bencoded list") + + args = json.loads(body) + + subreq_id = random(16) + endpoint = args['endpoint'] + query = args['query'] + pubkey = utils.decode_hex_or_b64(args['pubkey'], 33) + params = args['params'] + + # TOFIX: omq subrequest is signaled for execution by uwsgi.signal, which does not + # return a value. In the standard onion request implementation, the response is + # returned with the headers. In the OMQ onion request implementation, this is + # handled in omq.handle_proxied_omq_req, which receives a response and code from + # omq_auth.endpoint, passing them through the mule to omq.subreq_response + + make_omq_subreq( + subreq_id, + endpoint, + query, + pubkey, + msg.later(), + params, + client_reauth=True, # Because onion requests have auth headers on the *inside* + ) + + except Exception as e: + app.logger.warning("Invalid v4 onion request: {}".format(e)) + args = {'code': http.BAD_REQUEST, 'headers': {'content-type': 'text/plain; charset=utf-8'}} + data = b'Invalid v4 onion request' + + +def decrypt_onionreq(): + try: + return crypto.parse_junk(request.data) + except Exception as e: + app.logger.warning("Failed to decrypt onion request: {}".format(e)) + abort(http.BAD_REQUEST) + + +def handle_v4_onion_request(): + """ + Parse a v4 onion request. See handle_v4_onionreq_plaintext(). + """ + + # Some less-than-ideal decisions in the onion request protocol design means that we are stuck + # dealing with parsing the request body here in the internal format that is meant for storage + # server, but the *last* hop's decrypted, encoded data has to get shared by us (and is passed on + # to us in its raw, encoded form). It looks like this: + # + # [N][blob][json] + # + # where N is the size of blob (4 bytes, little endian), and json contains *both* the elements + # that were meant for the last hop (like our host/port/protocol) *and* the elements that *we* + # need to decrypt blob (specifically: "ephemeral_key" and, optionally, "enc_type" [which can be + # used to use xchacha20-poly1305 encryption instead of AES-GCM]). + # + # The parse_junk here takes care of decoding and decrypting this according to the fields *meant + # for us* in the json (which include things like the encryption type and ephemeral key): + try: + junk = crypto.parse_junk(request.data) + except RuntimeError as e: + app.logger.warning("Failed to decrypt onion request: {}".format(e)) + abort(http.BAD_REQUEST) + + # On the way back out we re-encrypt via the junk parser (which uses the ephemeral key and + # enc_type that were specified in the outer request). We then return that encrypted binary + # payload as-is back to the client which bounces its way through the SN path back to the client. + response = handle_v4onion_omqreq_plaintext(junk.payload) + return junk.transformReply(response) diff --git a/sogs/routes/rooms.py b/sogs/routes/rooms.py index 4487a71b..1cd2ceb0 100644 --- a/sogs/routes/rooms.py +++ b/sogs/routes/rooms.py @@ -1,5 +1,6 @@ from .. import config, db, http from ..model import room as mroom, exc, user as muser +from ..omq import omq_global from ..web import app from . import auth @@ -11,8 +12,8 @@ # Room-related routes, excluding retrieving/posting messages - rooms = Blueprint('rooms', __name__) +app.register_blueprint(rooms) def get_room_info(room): @@ -446,7 +447,6 @@ def set_permissions(room, sid): with db.transaction(): with user.check_blinding() as u: - if req.get('unschedule') is not False and any( p in perms for p in ('read', 'write', 'upload') ): diff --git a/sogs/routes/subrequest.py b/sogs/routes/subrequest.py index d856a5fd..c4cb5708 100644 --- a/sogs/routes/subrequest.py +++ b/sogs/routes/subrequest.py @@ -29,7 +29,7 @@ def make_subrequest( body - the bytes content of the body of a POST/PUT method. If specified then content_type will default to 'application/octet-stream'. json - a json value to dump as the body of the request. If specified then content_type will - default to 'applicaton/json'. + default to 'application/json'. user_reauth - if True then we allow user re-authentication on the subrequest based on its X-SOGS-* headers; if False (the default) then the user auth on the outer request is preserved (even if it was None) and inner request auth headers will be ignored. diff --git a/sogs/routes/users.py b/sogs/routes/users.py index 525ea158..9bc4c854 100644 --- a/sogs/routes/users.py +++ b/sogs/routes/users.py @@ -3,6 +3,7 @@ from ..model.user import User from ..web import app from . import auth +from ..omq import omq_global from flask import abort, jsonify, g, Blueprint, request @@ -10,6 +11,7 @@ users = Blueprint('users', __name__) +app.register_blueprint(users) def extract_rooms_or_global(req, admin=True): diff --git a/sogs/routes/views.py b/sogs/routes/views.py index 8ee8adcf..b20b018d 100644 --- a/sogs/routes/views.py +++ b/sogs/routes/views.py @@ -3,6 +3,8 @@ from .. import config, crypto, http from ..model.room import get_accessible_rooms from . import auth, converters # noqa: F401 +from ..omq import omq_global +from ..web import app from io import BytesIO @@ -18,6 +20,7 @@ views = Blueprint('views', __name__) +app.register_blueprint(views) @views.get("/") diff --git a/sogs/utils.py b/sogs/utils.py index bea930ac..a83a92be 100644 --- a/sogs/utils.py +++ b/sogs/utils.py @@ -3,6 +3,7 @@ from . import http import base64 + from flask import request, abort, Response import json from typing import Union, Tuple @@ -134,6 +135,10 @@ def get_int_param(name, default=None, *, required=False, min=None, max=None, tru return default try: + if val == "true" or val == "True" or val == True: + val = 1 + if val == "false" or val == "False" or val == False: + val = 0 val = int(val) except Exception: abort(http.BAD_REQUEST) diff --git a/tests/test_auth.py b/tests/test_auth.py index 923c111b..316224d7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -379,7 +379,6 @@ def test_auth_batch(client, db): def test_auth_legacy(client, db, admin, user, room): - # Make a legacy auth token to make sure it works as expected first, but also to make sure it # gets ignored when we use X-SOGS-*. raw_token = sogs.utils.make_legacy_token(admin.session_id) diff --git a/tests/test_files.py b/tests/test_files.py index e304b703..1f264e7c 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -156,7 +156,6 @@ def test_no_file_crosspost(client, room, room2, user, global_admin): def _file_upload(client, room, user, *, unsafe=False, utf=False, filename): - url_post = f"/room/{room.token}/file" file_content = random(1024) filename_escaped = urllib.parse.quote(filename.encode('utf-8')) diff --git a/tests/test_onion_requests.py b/tests/test_onion_requests.py index dd4d2fa5..3d519197 100644 --- a/tests/test_onion_requests.py +++ b/tests/test_onion_requests.py @@ -138,7 +138,6 @@ def decrypt_reply(data, *, v, enc_type): def test_v3(room, client): - # Construct an onion request for /room/test-room req = {'method': 'GET', 'endpoint': '/room/test-room'} data = build_payload(req, v=3, enc_type="xchacha20") @@ -154,7 +153,6 @@ def test_v3(room, client): def test_v3_authenticated(room, mod, client): - # Construct an onion request for /room/test-room req = {'method': 'GET', 'endpoint': '/room/test-room'} req['headers'] = auth.x_sogs(mod.ed_key, crypto.server_pubkey, req['method'], req['endpoint']) diff --git a/tests/test_room_routes.py b/tests/test_room_routes.py index e5980829..d1afe603 100644 --- a/tests/test_room_routes.py +++ b/tests/test_room_routes.py @@ -10,7 +10,6 @@ def test_list(client, room, room2, user, user2, admin, mod, global_mod, global_admin): - room2.default_write = False room2.default_upload = False @@ -750,17 +749,13 @@ def deleted_entry(id, seqno): *(deleted_entry(i, s) for i, s in ((2, 11), (4, 12), (5, 13), (8, 14), (9, 15))), ] assert get_and_clean_since(10) == [ - *(deleted_entry(i, s) for i, s in ((2, 11), (4, 12), (5, 13), (8, 14), (9, 15))), + *(deleted_entry(i, s) for i, s in ((2, 11), (4, 12), (5, 13), (8, 14), (9, 15))) ] assert get_and_clean_since(11) == [ - *(deleted_entry(i, s) for i, s in ((4, 12), (5, 13), (8, 14), (9, 15))), - ] - assert get_and_clean_since(13) == [ - *(deleted_entry(i, s) for i, s in ((8, 14), (9, 15))), - ] - assert get_and_clean_since(14) == [ - *(deleted_entry(i, s) for i, s in ((9, 15),)), + *(deleted_entry(i, s) for i, s in ((4, 12), (5, 13), (8, 14), (9, 15))) ] + assert get_and_clean_since(13) == [*(deleted_entry(i, s) for i, s in ((8, 14), (9, 15)))] + assert get_and_clean_since(14) == [*(deleted_entry(i, s) for i, s in ((9, 15),))] assert get_and_clean_since(15) == [] @@ -934,7 +929,6 @@ def room_json(): def test_posting(client, room, user, user2, mod, global_mod): - url_post = "/room/test-room/message" d, s = (utils.encode_base64(x) for x in (b"post 1", pad64("sig 1"))) r = sogs_post(client, url_post, {"data": d, "signature": s}, user) @@ -957,7 +951,6 @@ def test_posting(client, room, user, user2, mod, global_mod): def test_whisper_to(client, room, user, user2, mod, global_mod): - url_post = "/room/test-room/message" d, s = (utils.encode_base64(x) for x in (b"whisper 1", pad64("sig 1"))) p = {"data": d, "signature": s, "whisper_to": user2.session_id} @@ -1005,7 +998,6 @@ def test_whisper_to(client, room, user, user2, mod, global_mod): def test_whisper_mods(client, room, user, user2, mod, global_mod, admin): - url_post = "/room/test-room/message" d, s = (utils.encode_base64(x) for x in (b"whisper 1", pad64("sig 1"))) p = {"data": d, "signature": s, "whisper_mods": True} @@ -1045,7 +1037,6 @@ def test_whisper_mods(client, room, user, user2, mod, global_mod, admin): def test_whisper_both(client, room, user, user2, mod, admin): - # A whisper aimed at both a user *and* all mods (e.g. a warning to a user) url_post = "/room/test-room/message" @@ -1138,7 +1129,6 @@ def test_whisper_both(client, room, user, user2, mod, admin): def test_edits(client, room, user, user2, mod, global_admin): - url_post = "/room/test-room/message" d, s = (utils.encode_base64(x) for x in (b"post 1", pad64("sig 1"))) r = sogs_post(client, url_post, {"data": d, "signature": s}, user) @@ -1401,7 +1391,6 @@ def test_set_room_perms(client, room, user, mod): def test_set_room_perm_futures(client, room, user, mod): - r = sogs_post( client, '/sequence', diff --git a/tests/test_rooms.py b/tests/test_rooms.py index 59b4469e..0f21311c 100644 --- a/tests/test_rooms.py +++ b/tests/test_rooms.py @@ -9,7 +9,6 @@ def test_create(room, room2): - r3 = Room.create('Test_Room-3', name='Test room 3', description='Test suite testing room3') rooms = get_rooms() @@ -36,7 +35,6 @@ def test_create(room, room2): def test_token_insensitive(room): - r = Room.create('Test_Ro-om', name='TR2', description='Test suite testing room2') r_a = Room(token='Test_Ro-om') @@ -92,7 +90,6 @@ def test_info(room): def test_updates(room): - assert room.message_sequence == 0 and room.info_updates == 0 and room.name == 'Test room' room.name = 'Test Room' @@ -118,7 +115,6 @@ def test_updates(room): def test_permissions(room, user, user2, mod, admin, global_mod, global_admin): - # Public permissions: assert not room.check_permission(admin=True) assert not room.check_permission(moderator=True) @@ -386,7 +382,6 @@ def test_bans(room, user, user2, mod, admin, global_mod, global_admin): def test_mods(room, user, user2, mod, admin, global_mod, global_admin): - room.set_moderator(user, added_by=admin) assert room.check_moderator(user) assert not room.check_admin(user) @@ -458,7 +453,6 @@ def test_mods(room, user, user2, mod, admin, global_mod, global_admin): def test_upload(room, user): - import os file = File(id=room.upload_file(content=b'abc', uploader=user, filename="abc.txt", lifetime=30)) @@ -489,7 +483,6 @@ def test_upload(room, user): def test_upload_expiry(room, user): - import os file = File(id=room.upload_file(content=b'abc', uploader=user, filename="abc.txt", lifetime=-1)) @@ -512,7 +505,6 @@ def test_upload_expiry(room, user): def test_image(room, user): - assert room.image is None fid = room.upload_file(content=b'abc', uploader=user, filename="abc.txt") diff --git a/tests/test_routes_general.py b/tests/test_routes_general.py index 9744b494..169d215c 100644 --- a/tests/test_routes_general.py +++ b/tests/test_routes_general.py @@ -134,7 +134,6 @@ def batch_test_endpoint4(): def test_batch(client): - d1, b1_exp = batch_data() b1 = client.post("/batch", json=d1) assert b1.json == b1_exp diff --git a/tests/test_user_routes.py b/tests/test_user_routes.py index 9fe54063..ea2fcb43 100644 --- a/tests/test_user_routes.py +++ b/tests/test_user_routes.py @@ -5,7 +5,6 @@ def test_global_mods(client, room, room2, user, user2, mod, admin, global_admin, global_mod): - assert not room2.check_moderator(user) assert not room2.check_moderator(user2) @@ -167,7 +166,6 @@ def test_global_mods(client, room, room2, user, user2, mod, admin, global_admin, def test_room_mods(client, room, room2, user, user2, mod, admin, global_admin, global_mod): - # Track expected info_updates values; the initial values are because creating the mod/admin/etc. # fixtures imported here perform db modifications that trigger updates (2 global mods + 2 mods # of `room`):