From c5e4afeabfd7d943e570d66053c63471e12cb63b Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 31 Jan 2023 15:58:42 -0800 Subject: [PATCH 01/11] initial commit: barebones bot model plus bot_mode implementation details --- sogs/model/bot.py | 70 +++++++++++++++++++++++++++++++++++++++++ sogs/model/room.py | 10 ++++++ sogs/routes/messages.py | 52 +++++++++++++++++------------- 3 files changed, 110 insertions(+), 22 deletions(-) create mode 100644 sogs/model/bot.py diff --git a/sogs/model/bot.py b/sogs/model/bot.py new file mode 100644 index 00000000..f2c29ebf --- /dev/null +++ b/sogs/model/bot.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from .. import crypto, db, config +from ..db import query +from ..web import app +from .exc import NoSuchUser, BadPermission +from sogs.model.user import User +from .exc import (InvalidData) + +from typing import Optional, List +import time +import contextlib + + +class Bot: + """ + Class representing a simple bot to manage open group server + + Object Properties: + id - database primary key for user row + session_id - jex encoded session_id of the bot + banned - default to false + global_admin - default to true for bot + global_moderator - default to true for bot + visible_mod - default to true for bot + """ + + def __init__( + self, + row = None, + *, + id: Optional[int] = None, + session_id: Optional[int] = None) -> None: + + # immutable attributes + self._banned = False + self._global_admin = False + self._global_moderator = False + self._visible_mod = False + + # operational attributes + self.current_message = None + self.nlp_model = None + self.language = "English" + self.word_blacklist = ['placeholderlist', 'until', 'we', 'decide', 'naughty', 'words'] + + + def __setattr__(self, __name: str, __value) -> None: + if __name in ['_banned', '_global_admin', '_global_moderator', '_visible_mod']: + raise AttributeError("Cannot modify bots") + else: + setattr(self, __name, __value) + + def __delattr__(self, __name: str) -> None: + if __name in ['_banned', '_global_admin', '_global_moderator', '_visible_mod']: + raise AttributeError("Cannot modify bots") + else: + delattr(self, __name) + + def receive_message(self, + user: User, + data: bytes, + sig: bytes, + *, + files: List[int] = []): + + if data is None or sig is None or len(sig) != 64: + raise InvalidData() + + \ No newline at end of file diff --git a/sogs/model/room.py b/sogs/model/room.py index 7217b116..3910f81b 100644 --- a/sogs/model/room.py +++ b/sogs/model/room.py @@ -17,6 +17,7 @@ PostRateLimited, InvalidData, ) +from model.bot import Bot import os import random @@ -71,6 +72,9 @@ class Room: default_accessible - True if default user permissions include accessible permission default_write - True if default user permissions includes write permission default_upload - True if default user permissions includes file upload permission + + NEW: + bot_mode - true if bot moderator is actively on patrol """ def __init__(self, row=None, *, id=None, token=None): @@ -79,6 +83,12 @@ def __init__(self, row=None, *, id=None, token=None): looking up this raises a NoSuchRoom if no room with that token/id exists. """ self._refresh(id=id, token=token, row=row) + self.bot_mode = False + self.bot = None + + def add_bot(self, _bot: Bot): + self.bot = _bot + self.bot_mode = True def _refresh(self, *, id=None, token=None, row=None, perms=False): """ diff --git a/sogs/routes/messages.py b/sogs/routes/messages.py index 34b65447..e9054d8c 100644 --- a/sogs/routes/messages.py +++ b/sogs/routes/messages.py @@ -1,5 +1,6 @@ from .. import http, utils from . import auth +from model.room import Room from flask import abort, jsonify, g, Blueprint, request @@ -15,7 +16,7 @@ def qs_reactors(): @messages.get("/room//messages/since/") @auth.read_required -def messages_since(room, seqno): +def messages_since(room: Room, seqno): """ Retrieves message *updates* from a room. This is the main message polling endpoint in SOGS. @@ -99,7 +100,7 @@ def messages_since(room, seqno): @messages.get("/room//messages/before/") @auth.read_required -def messages_before(room, msg_id): +def messages_before(room: Room, msg_id): """ Retrieves messages from the room preceding a given id. @@ -142,7 +143,7 @@ def messages_before(room, msg_id): @messages.get("/room//messages/recent") @auth.read_required -def messages_recent(room): +def messages_recent(room: Room): """ Retrieves recent messages posted to this room. @@ -181,7 +182,7 @@ def messages_recent(room): @messages.get("/room//message/") @auth.read_required -def message_single(room, msg_id): +def message_single(room: Room, msg_id): """ Returns a single message by ID. @@ -310,7 +311,7 @@ def message_single(room, msg_id): @messages.post("/room//message") @auth.user_required -def post_message(room): +def post_message(room: Room): """ Posts a new message to a room. @@ -359,21 +360,28 @@ def post_message(room): """ req = request.json - msg = room.add_post( - g.user, - data=utils.decode_base64(req.get('data')), - sig=utils.decode_base64(req.get('signature')), - whisper_to=req.get('whisper_to'), - whisper_mods=bool(req.get('whisper_mods')), - files=[int(x) for x in req.get('files', [])], - ) + if room.bot_mode: + room.bot.receive_message( + user = g.user, + data = utils.decode_base64(req.get('data')), + sig = utils.decode_base64(req.get('signature')), + files = [int(x) for x in req.get('files', [])]) + else: + msg = room.add_post( + g.user, + data=utils.decode_base64(req.get('data')), + sig=utils.decode_base64(req.get('signature')), + whisper_to=req.get('whisper_to'), + whisper_mods=bool(req.get('whisper_mods')), + files=[int(x) for x in req.get('files', [])], + ) return utils.jsonify_with_base64(msg), http.CREATED @messages.put("/room//message/") @auth.user_required -def edit_message(room, msg_id): +def edit_message(room: Room, msg_id): """ Edits a message, replacing its existing content with new content and a new signature. @@ -420,7 +428,7 @@ def edit_message(room, msg_id): @messages.delete("/room//message/") @auth.user_required -def remove_message(room, msg_id): +def remove_message(room: Room, msg_id): """ Remove a message by its message id @@ -447,7 +455,7 @@ def remove_message(room, msg_id): @messages.post("/room//pin/") -def message_pin(room, msg_id): +def message_pin(room: Room, msg_id): """ Adds a pinned message to this room. @@ -490,7 +498,7 @@ def message_pin(room, msg_id): @messages.post("/room//unpin/") -def message_unpin(room, msg_id): +def message_unpin(room: Room, msg_id): """ Remove a message from this room's pinned message list. @@ -524,7 +532,7 @@ def message_unpin(room, msg_id): @messages.post("/room//unpin/all") -def message_unpin_all(room): +def message_unpin_all(room: Room): """ Removes *all* pinned messages from this room. @@ -553,7 +561,7 @@ def message_unpin_all(room): @messages.put("/room//reaction//") @auth.user_required @auth.read_required -def message_react(room, msg_id, reaction): +def message_react(room: Room, msg_id, reaction): """ Adds a reaction to the given message in this room. The user must have read access in the room. @@ -606,7 +614,7 @@ def message_react(room, msg_id, reaction): @messages.delete("/room//reaction//") @auth.user_required @auth.read_required -def message_unreact(room, msg_id, reaction): +def message_unreact(room: Room, msg_id, reaction): """ Removes a reaction from a post this room. The user must have read access in the room. This only removes the user's own reaction but does not affect the reactions of other users. @@ -642,7 +650,7 @@ def message_unreact(room, msg_id, reaction): @messages.delete("/room//reactions//") @messages.delete("/room//reactions/") @auth.mod_required -def message_delete_reactions(room, msg_id, reaction=None): +def message_delete_reactions(room: Room, msg_id, reaction=None): """ Removes all reactions of all users from a post in this room. The calling must have moderator permissions in the room. This endpoint can either remove a single reaction (e.g. remove all 🍆 @@ -677,7 +685,7 @@ def message_delete_reactions(room, msg_id, reaction=None): @messages.get("/room//reactors//") @auth.read_required -def message_get_reactors(room, msg_id, reaction): +def message_get_reactors(room: Room, msg_id, reaction): """ Returns the list of all reactors who have added a particular reaction to a particular message. From b28e5601b86f9aa39e4ec2c511054601398d18db Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 Feb 2023 11:12:49 -0800 Subject: [PATCH 02/11] created simplefilter class to encapsulate all filtering functionality. made bots active by default with toggle moderation mode active (moderating) or passive (filter only). rooms being moderated by bot are stored in dict as room:active_mode(bool) --- conftest.py | 2 +- contrib/auth-example.py | 1 - contrib/pg-import.py | 2 - sogs/__main__.py | 4 +- sogs/migrations/file_message.py | 1 - sogs/migrations/import_hacks.py | 2 +- sogs/migrations/reactions.py | 2 - sogs/migrations/v_0_1_x.py | 5 - sogs/model/bot.py | 239 ++++++++++++++-- sogs/model/filter.py | 212 ++++++++++++++ sogs/model/post.py | 6 +- sogs/model/room.py | 472 +++----------------------------- sogs/postfork.py | 1 - sogs/routes/legacy.py | 1 - sogs/routes/messages.py | 24 +- sogs/routes/rooms.py | 1 - tests/test_auth.py | 1 - tests/test_files.py | 1 - tests/test_onion_requests.py | 2 - tests/test_room_routes.py | 19 +- tests/test_rooms.py | 8 - tests/test_routes_general.py | 1 - tests/test_user_routes.py | 2 - 23 files changed, 493 insertions(+), 516 deletions(-) create mode 100644 sogs/model/filter.py diff --git a/conftest.py b/conftest.py index e7343774..9c3eeaf2 100644 --- a/conftest.py +++ b/conftest.py @@ -267,7 +267,7 @@ def blind_user2(db): @pytest.fixture def no_rate_limit(): """Disables post rate limiting for the test""" - import sogs.model.room as mroom + from sogs.model.room import Room as mroom saved = (mroom.rate_limit_size, mroom.rate_limit_interval) mroom.rate_limit_size, mroom.rate_limit_interval = None, None diff --git a/contrib/auth-example.py b/contrib/auth-example.py index 0e6767b8..ed52376a 100755 --- a/contrib/auth-example.py +++ b/contrib/auth-example.py @@ -71,7 +71,6 @@ def get_signing_headers( body, blinded: bool = True, ): - assert len(server_pk) == 32 assert len(nonce) == 16 diff --git a/contrib/pg-import.py b/contrib/pg-import.py index 61c7a1a3..df832ae8 100755 --- a/contrib/pg-import.py +++ b/contrib/pg-import.py @@ -86,7 +86,6 @@ with pgsql.transaction(): - curin = old.cursor() curout = pgsql.cursor() @@ -131,7 +130,6 @@ curout.execute("ALTER TABLE rooms DROP CONSTRAINT room_image_fk") def copy(table): - cols = [r['name'] for r in curin.execute(f"PRAGMA table_info({table})")] if not cols: raise RuntimeError(f"Expected table {table} does not exist in sqlite db") diff --git a/sogs/__main__.py b/sogs/__main__.py index d897949e..dc027c0b 100644 --- a/sogs/__main__.py +++ b/sogs/__main__.py @@ -400,7 +400,6 @@ def parse_and_set_perm_flags(flags, perm_setting): sys.exit(2) elif update_room: - rooms = [] all_rooms = False global_rooms = False @@ -577,8 +576,7 @@ def parse_and_set_perm_flags(flags, perm_setting): if args.name is not None: if global_rooms or all_rooms: print( - "Error: --rooms cannot be '+' or '*' (i.e. global/all) with --name", - file=sys.stderr, + "Error: --rooms cannot be '+' or '*' (i.e. global/all) with --name", file=sys.stderr ) sys.exit(1) diff --git a/sogs/migrations/file_message.py b/sogs/migrations/file_message.py index 14eac2e5..d8680644 100644 --- a/sogs/migrations/file_message.py +++ b/sogs/migrations/file_message.py @@ -3,7 +3,6 @@ def migrate(conn, *, check_only): - from .. import db fix_fk = False diff --git a/sogs/migrations/import_hacks.py b/sogs/migrations/import_hacks.py index 1556cb70..2db9d46a 100644 --- a/sogs/migrations/import_hacks.py +++ b/sogs/migrations/import_hacks.py @@ -43,7 +43,7 @@ def migrate(conn, *, check_only): rows = conn.execute( "SELECT room, old_message_id_max, message_id_offset FROM room_import_hacks" ) - for (room, id_max, offset) in rows: + for room, id_max, offset in rows: db.ROOM_IMPORT_HACKS[room] = (id_max, offset) if not db.HAVE_FILE_ID_HACKS and 'room_import_hacks' not in db.metadata.tables: diff --git a/sogs/migrations/reactions.py b/sogs/migrations/reactions.py index acff1aa7..32acecda 100644 --- a/sogs/migrations/reactions.py +++ b/sogs/migrations/reactions.py @@ -3,7 +3,6 @@ def migrate(conn, *, check_only): - from .. import db if 'user_reactions' in db.metadata.tables: @@ -177,7 +176,6 @@ def migrate(conn, *, check_only): ) else: # postgresql - if 'seqno_data' not in db.metadata.tables['messages'].c: conn.execute( """ diff --git a/sogs/migrations/v_0_1_x.py b/sogs/migrations/v_0_1_x.py index b59c1ef3..740af695 100644 --- a/sogs/migrations/v_0_1_x.py +++ b/sogs/migrations/v_0_1_x.py @@ -39,7 +39,6 @@ def sqlite_connect_readonly(path): def import_from_0_1_x(conn): - from .. import config, db, utils # Old database database.db is a single table database containing just the list of rooms: @@ -109,7 +108,6 @@ def ins_user(session_id): ) with sqlite_connect_readonly(room_db_path) as rconn: - # Messages were stored in this: # # CREATE TABLE IF NOT EXISTS messages ( @@ -235,7 +233,6 @@ def ins_user(session_id): and data in (None, "deleted") and signature in (None, "deleted") ): - # Deleted message; we still need to insert a tombstone for it, and copy the # deletion id as the "seqno" field. (We do this with a second query # because the first query is going to trigger an automatic update of the @@ -340,7 +337,6 @@ def ins_user(session_id): n_files = rconn.execute("SELECT COUNT(*) FROM files").fetchone()[0] for file_id, timestamp in rconn.execute("SELECT id, timestamp FROM files"): - # file_id is an integer value but stored in a TEXT field, of course. file_id = int(file_id) @@ -507,7 +503,6 @@ def ins_user(session_id): """, (import_cutoff,), ): - ins_user(session_id) db.query( """ diff --git a/sogs/model/bot.py b/sogs/model/bot.py index f2c29ebf..e3637398 100644 --- a/sogs/model/bot.py +++ b/sogs/model/bot.py @@ -3,13 +3,17 @@ from .. import crypto, db, config from ..db import query from ..web import app -from .exc import NoSuchUser, BadPermission -from sogs.model.user import User -from .exc import (InvalidData) +from .exc import BadPermission, PostRateLimited +from .. import utils +from ..omq import send_mule +from .user import User +from .room import Room +from .message import Message +from .filter import SimpleFilter +from .exc import InvalidData -from typing import Optional, List +from typing import Optional, List, Union import time -import contextlib class Bot: @@ -18,7 +22,11 @@ class Bot: Object Properties: id - database primary key for user row - session_id - jex encoded session_id of the bot + status - bot active(T)/passive(F) moderation status (key = room, value = bool) + rooms - reference to room(s) in which bot is active + filter - reference to filter object paired with bot + perm_cache - dict storing user info/status (synced from all rooms patrolled) + session_id - hex encoded session_id of the bot banned - default to false global_admin - default to true for bot global_moderator - default to true for bot @@ -27,44 +35,233 @@ class Bot: def __init__( self, - row = None, + _rooms: List[Room], + row=None, *, id: Optional[int] = None, - session_id: Optional[int] = None) -> None: - + session_id: Optional[int] = None, + ) -> None: # immutable attributes - self._banned = False - self._global_admin = False - self._global_moderator = False - self._visible_mod = False + self._banned: bool = False + self._global_admin: bool = False + self._global_moderator: bool = False + self._visible_mod: bool = False + self.id = id # operational attributes - self.current_message = None + self.rooms: List[Room] = _rooms + self.status = {r: False for r in self.rooms} + self.filter = SimpleFilter(_bot=self) + self.perm_cache = {} + self.current_message: Message = None self.nlp_model = None - self.language = "English" + self.language = 'English' self.word_blacklist = ['placeholderlist', 'until', 'we', 'decide', 'naughty', 'words'] - def __setattr__(self, __name: str, __value) -> None: if __name in ['_banned', '_global_admin', '_global_moderator', '_visible_mod']: - raise AttributeError("Cannot modify bots") + raise AttributeError('Cannot modify bots') else: setattr(self, __name, __value) def __delattr__(self, __name: str) -> None: if __name in ['_banned', '_global_admin', '_global_moderator', '_visible_mod']: - raise AttributeError("Cannot modify bots") + raise AttributeError('Cannot modify bots') else: delattr(self, __name) - def receive_message(self, + def _link_room(self, _room: Room): + self.rooms.append(_room) + self.filter.rooms.append(_room) + + def _unlink_room(self, _room: Room): + self.rooms.remove(_room) + + if not self.rooms: + delete_bot_function_goes_here = True + + def _refresh_cache(self): + for r in self.rooms: + self.perm_cache = self.perm_cache | r._perm_cache + + def check_permission_for( + self, + room: Room, + user: Optional[User] = None, + *, + admin=False, + moderator=False, + read=False, + accessible=False, + write=False, + upload=False, + ): + """ + Checks whether `user` has the required permissions for this room and isn't banned. Returns + True if the user satisfies the permissions, False otherwise. If no user is provided then + permissions are checked against the room's defaults. + + Looked up permissions are cached within the Room instance so that looking up the same user + multiple times (i.e. from multiple parts of the code) does not re-query the database. + + Named arguments are as follows: + - admin -- if true then the user must have admin access to the room + - moderator -- if true then the user must have moderator (or admin) access to the room + - read -- if true then the user must have read access + - accessible -- if true then the user must have accessible access; note that this permission + is satisfied by *either* the `accessible` or `read` database flags (that is: read implies + accessible). + - write -- if true then the user must have write access + - upload -- if true then the user must have upload access; this should usually be combined + with write=True. + + You can specify multiple permissions as True, in which case all must be satisfied. If you + specify no permissions as required then the check only checks whether a user is banned but + otherwise requires no specific permission. + """ + + if user is None: + is_banned, can_read, can_access, can_write, can_upload, is_mod, is_admin = ( + False, + bool(room.default_read), + bool(room.default_accessible), + bool(room.default_write), + bool(room.default_upload), + False, + False, + ) + else: + if user.id not in self._perm_cache: + row = query( + """ + SELECT banned, read, accessible, write, upload, moderator, admin + FROM user_permissions + WHERE room = :r AND "user" = :u + """, + r=self.id, + u=user.id, + ).first() + self._perm_cache[user.id] = [bool(c) for c in row] + + ( + is_banned, + can_read, + can_access, + can_write, + can_upload, + is_mod, + is_admin, + ) = self._perm_cache[user.id] + + # Shortcuts for check_permission calls + def check_unbanned(self, room: Room, user: Optional[User]): + return self.check_permission_for(room, user) + + def check_read(self, room: Room, user: Optional[User] = None): + return self.check_permission_for(room, user, read=True) + + def check_accessible(self, room: Room, user: Optional[User] = None): + return self.check_permission_for(room, user, accessible=True) + + def check_write(self, room: Room, user: Optional[User] = None): + return self.check_permission_for(room, user, write=True) + + def check_upload(self, room: Room, user: Optional[User] = None): + """Checks for both upload *and* write permission""" + return self.check_permission_for(room, user, write=True, upload=True) + + def check_moderator(self, room: Room, user: Optional[User]): + return self.check_permission_for(room, user, moderator=True) + + def check_admin(self, room: Room, user: Optional[User]): + return self.check_permission_for(room, user, admin=True) + + def receive_message( + self, + room: Room, user: User, data: bytes, sig: bytes, *, - files: List[int] = []): + whisper_to: Optional[Union[User, str]] = None, + whisper_mods: bool = False, + files: List[int] = [], + ): + if not self.check_write(user): + raise BadPermission() if data is None or sig is None or len(sig) != 64: raise InvalidData() - \ No newline at end of file + whisper_mods = bool(whisper_mods) + if (whisper_to or whisper_mods) and not self.check_moderator(user): + app.logger.warning(f"Cannot post a whisper to {room}: {user} is not a moderator") + raise BadPermission() + + if whisper_to and not isinstance(whisper_to, User): + whisper_to = User(session_id=whisper_to, autovivify=True, touch=False) + + filtered = self.filter.read_message(user, data, room) + + with db.transaction(): + if room.rate_limit_size and not self.check_admin(user): + since_limit = time.time() - room.rate_limit_interval + recent_count = query( + """ + SELECT COUNT(*) FROM messages + WHERE room = :r AND "user" = :u AND posted >= :since + """, + r=self.id, + u=user.id, + since=since_limit, + ).first()[0] + + if recent_count >= room.rate_limit_size: + raise PostRateLimited() + + data_size = len(data) + unpadded_data = utils.remove_session_message_padding(data) + + msg_id = db.insert_and_get_pk( + """ + INSERT INTO messages + (room, "user", data, data_size, signature, filtered, whisper, whisper_mods) + VALUES + (:r, :u, :data, :data_size, :signature, :filtered, :whisper, :whisper_mods) + """, + "id", + r=room.id, + u=user.id, + data=unpadded_data, + data_size=data_size, + signature=sig, + filtered=filtered is not None, + whisper=whisper_to.id if whisper_to else None, + whisper_mods=whisper_mods, + ) + + if files: + # Take ownership of any uploaded files attached to the post: + room._own_files(msg_id, files, user) + + assert msg_id is not None + row = query("SELECT posted, seqno FROM messages WHERE id = :m", m=msg_id).first() + msg = { + 'id': msg_id, + 'session_id': user.session_id, + 'posted': row[0], + 'seqno': row[1], + 'data': data, + 'signature': sig, + 'reactions': {}, + } + if filtered is not None: + msg['filtered'] = True + if whisper_to or whisper_mods: + msg['whisper'] = True + msg['whisper_mods'] = whisper_mods + if whisper_to: + msg['whisper_to'] = whisper_to.session_id + + send_mule("message_posted", msg["id"]) + return msg diff --git a/sogs/model/filter.py b/sogs/model/filter.py new file mode 100644 index 00000000..edbdf320 --- /dev/null +++ b/sogs/model/filter.py @@ -0,0 +1,212 @@ +from __future__ import annotations +from .. import config, crypto, db, utils, session_pb2 as protobuf +import random + +from .. import crypto, db, config +from ..db import query +from ..hashing import blake2b +from ..web import app +from nacl.signing import SigningKey +from .exc import NoSuchUser, BadPermission, PostRejected +from sogs.model.user import User +from sogs.model.bot import Bot +from sogs.model.room import Room, alphabet_filter_patterns +from sogs.model.post import Post +from .exc import InvalidData + +from typing import Optional, List, Union +import time +import contextlib + + +class SimpleFilter: + """ + Class representing a simple word filter searching for naughty words + + Object Properties: + bot - bot this filter is servicing + current_message - reference to current data being analyzed + """ + + def __init__(self, _bot: Bot, _room: Room = None): + self.bot: Bot = _bot + self.current_message: Post = None + + def filtering(self): + settings = { + 'profanity_filter': config.PROFANITY_FILTER, + 'profanity_silent': config.PROFANITY_SILENT, + 'alphabet_filters': config.ALPHABET_FILTERS, + 'alphabet_silent': config.ALPHABET_SILENT, + } + if self.token in config.ROOM_OVERRIDES: + for k in ( + 'profanity_filter', + 'profanity_silent', + 'alphabet_filters', + 'alphabet_silent', + ): + if k in config.ROOM_OVERRIDES[self.token]: + settings[k] = config.ROOM_OVERRIDES[self.token][k] + return settings + + def filter_should_reply(self, filter_type, filter_lang, room: Room): + """If the settings say we should reply to a filter, this returns a tuple of + + (profile name, message format, whisper) + + where profile name is the name we should use in the reply, message format is a string with + substitutions ready, and whisper is True/False depending on whether it should be whispered + to the user (True) or public (False). + + If we shouldn't reply this returns (None, None, None) + """ + + if not config.FILTER_SETTINGS: + return (None, None, None) + + reply_format = None + profile_name = 'SOGS' + public = False + + # Precedences from least to most specific so that we load values from least specific first + # then overwrite them if we find a value in a more specific section + room_precedence = ('*', room.token) + filter_precedence = ('*', filter_type, filter_lang) if filter_lang else ('*', filter_type) + + for r in room_precedence: + s1 = config.FILTER_SETTINGS.get(r) + if s1 is None: + continue + for f in filter_precedence: + settings = s1.get(f) + if settings is None: + continue + + rf = settings.get('reply') + pn = settings.get('profile_name') + pb = settings.get('public') + if rf is not None: + reply_format = random.choice(rf) + if pn is not None: + profile_name = pn + if pb is not None: + public = pb + + return (reply_format, profile_name, public) + + def read_message(self, user: User, data: bytes, room: Room): + """ + Checks a message for disallowed alphabets and profanity (if the profanity + filter is enabled). + + - Returns None if this message passes (i.e. didn't trigger any filter, or is + being posted by an admin to whom the filters don't apply). + + - Returns a callback if the message fails but should be silently accepted. The callback + should be called (with no arguments) *after* the filtered message is inserted into the db. + + - Throws PostRejected if the message should be rejected (and rejection passed back to the + user) + """ + + if not data: + raise ValueError('No message data passed to filter') + + self.current_message = Post(raw=data) + + if not config.FILTER_MODS and self.check_moderator(user): + return None + + filt = self.filtering() + alphabets = filt['alphabet_filters'] + for lang, pattern in alphabet_filter_patterns: + if lang not in alphabets: + continue + + if not pattern.search(self.current_message().text): + continue + + # Filter it! + filter_type, filter_lang = 'alphabet', lang + break + + if not filter_type and filt['profanity_filter']: + import better_profanity + + for part in (self.current_message().text, self.current_message().username): + if better_profanity.profanity.contains_profanity(part): + filter_type = 'profanity' + break + + if not filter_type: + return None + + silent = filt[filter_type + '_silent'] + + msg_fmt, prof_name, pub = self.filter_should_reply(filter_type, filter_lang) + if msg_fmt: + pbmsg = protobuf.Content() + body = msg_fmt.format( + profile_name=( + user.session_id + if self.current_message().username is None + else self.current_message().username + ), + profile_at="@" + user.session_id, + room_name=self.name, + room_token=self.token, + ).encode() + pbmsg.dataMessage.body = body + pbmsg.dataMessage.timestamp = int(time.time() * 1000) + pbmsg.dataMessage.profile.displayName = prof_name + + # Add two bytes padding so that session doesn't get confused by a lack of padding + pbmsg = pbmsg.SerializeToString() + b'\x80\x00' + + # Make a fake signing key based on prof_name and the server privkey (so that different + # names use different keys; otherwise the bot names overwrite each other in Session + # clients when a later message has a new profile name). + global filter_privkeys + if prof_name in room.filter_privkeys: + signingkey = filter_privkeys[prof_name] + else: + signingkey = SigningKey( + blake2b( + prof_name.encode() + crypto.server_signkey.encode(), key=b'sogsfiltering' + ) + ) + filter_privkeys[prof_name] = signingkey + + sig = signingkey.sign(pbmsg).signature + server_fake_user = User( + session_id='15' + signingkey.verify_key.encode().hex(), autovivify=True, touch=False + ) + + def insert_reply(): + query( + """ + INSERT INTO messages + (room, "user", data, data_size, signature, whisper) + VALUES + (:r, :u, :data, :data_size, :signature, :whisper) + """, + r=room.id, + u=server_fake_user.id, + data=pbmsg[:-2], + data_size=len(pbmsg), + signature=sig, + whisper=None if pub else user.id, + ) + + if filt[filter_type + '_silent']: + # Defer the insertion until after the filtered row gets inserted + return insert_reply + else: + insert_reply() + + elif silent: + return lambda: None + + # TODO: can we send back some error code that makes Session not retry? + raise PostRejected(f"filtration rejected message ({filter_type})") diff --git a/sogs/model/post.py b/sogs/model/post.py index b0d3a5a0..463ad1fc 100644 --- a/sogs/model/post.py +++ b/sogs/model/post.py @@ -18,17 +18,17 @@ def __init__(self, raw=None, *, user=None, text=None): @property def text(self): - """ accessor for the post body """ + """accessor for the post body""" return self._proto.body @property def username(self): - """ accessor for the username of the post's author """ + """accessor for the username of the post's author""" if self.profile is None: return return self.profile.displayName @property def profile(self): - """ accessor for the user profile data containing things like username etc """ + """accessor for the user profile data containing things like username etc""" return self._proto.profile diff --git a/sogs/model/room.py b/sogs/model/room.py index 3910f81b..8d829cc4 100644 --- a/sogs/model/room.py +++ b/sogs/model/room.py @@ -18,6 +18,7 @@ InvalidData, ) from model.bot import Bot +from model.filter import SimpleFilter import os import random @@ -27,12 +28,6 @@ from typing import Optional, Union, List -# TODO: These really should be room properties, not random global constants (these -# are carried over from old SOGS). -rate_limit_size = 5 -rate_limit_interval = 16.0 - - # Character ranges for different filters. This is ordered because some are subsets of each other # (e.g. persian is a subset of the arabic character range). alphabet_filter_patterns = [ @@ -72,23 +67,30 @@ class Room: default_accessible - True if default user permissions include accessible permission default_write - True if default user permissions includes write permission default_upload - True if default user permissions includes file upload permission - + NEW: - bot_mode - true if bot moderator is actively on patrol + default_bot - default_bot object reference + active_bot - true if default_bot moderator is actively moderating or false if passively filtering """ - def __init__(self, row=None, *, id=None, token=None): + def __init__(self, row=None, *, id=None, token=None, bot: Bot = None): """ Constructs a room from a pre-retrieved row *or* via lookup of a room token or id. When looking up this raises a NoSuchRoom if no room with that token/id exists. """ self._refresh(id=id, token=token, row=row) - self.bot_mode = False - self.bot = None + self.active_bot: bool = False + self.default_bot: Bot = bot if bot else Bot(_rooms=[self]) + self.rate_limit_size = 5 + self.rate_limit_interval = 16.0 + + def activate_moderation(self): + self.default_bot.status[self] = True + self.active_bot = True - def add_bot(self, _bot: Bot): - self.bot = _bot - self.bot_mode = True + def filter_only(self): + self.default_bot.status[self] = False + self.active_bot = False def _refresh(self, *, id=None, token=None, row=None, perms=False): """ @@ -157,6 +159,8 @@ def _refresh(self, *, id=None, token=None, row=None, perms=False): if perms or not hasattr(self, '_perm_cache'): self._perm_cache = {} + self.default_bot._refresh_cache() + def __str__(self): """Returns `Room[token]` when converted to a str""" return f"Room[{self.token}]" @@ -436,114 +440,6 @@ def active_users_last(self, cutoff: float): since=time.time() - cutoff, ).first()[0] - def check_permission( - self, - user: Optional[User] = None, - *, - admin=False, - moderator=False, - read=False, - accessible=False, - write=False, - upload=False, - ): - """ - Checks whether `user` has the required permissions for this room and isn't banned. Returns - True if the user satisfies the permissions, False otherwise. If no user is provided then - permissions are checked against the room's defaults. - - Looked up permissions are cached within the Room instance so that looking up the same user - multiple times (i.e. from multiple parts of the code) does not re-query the database. - - Named arguments are as follows: - - admin -- if true then the user must have admin access to the room - - moderator -- if true then the user must have moderator (or admin) access to the room - - read -- if true then the user must have read access - - accessible -- if true then the user must have accessible access; note that this permission - is satisfied by *either* the `accessible` or `read` database flags (that is: read implies - accessible). - - write -- if true then the user must have write access - - upload -- if true then the user must have upload access; this should usually be combined - with write=True. - - You can specify multiple permissions as True, in which case all must be satisfied. If you - specify no permissions as required then the check only checks whether a user is banned but - otherwise requires no specific permission. - """ - - if user is None: - is_banned, can_read, can_access, can_write, can_upload, is_mod, is_admin = ( - False, - bool(self.default_read), - bool(self.default_accessible), - bool(self.default_write), - bool(self.default_upload), - False, - False, - ) - else: - if user.id not in self._perm_cache: - row = query( - """ - SELECT banned, read, accessible, write, upload, moderator, admin - FROM user_permissions - WHERE room = :r AND "user" = :u - """, - r=self.id, - u=user.id, - ).first() - self._perm_cache[user.id] = [bool(c) for c in row] - - ( - is_banned, - can_read, - can_access, - can_write, - can_upload, - is_mod, - is_admin, - ) = self._perm_cache[user.id] - - if is_admin: - return True - if admin: - return False - if is_mod: - return True - if moderator: - return False - return ( - not is_banned - and (not accessible or can_access or can_read) - and (not read or can_read) - and (not write or can_write) - and (not upload or can_upload) - ) - - # Shortcuts for check_permission calls - - def check_unbanned(self, user: Optional[User]): - return self.check_permission(user) - - def check_read(self, user: Optional[User] = None): - return self.check_permission(user, read=True) - - def check_accessible(self, user: Optional[User] = None): - return self.check_permission(user, accessible=True) - - def check_write(self, user: Optional[User] = None): - return self.check_permission(user, write=True) - - def check_upload(self, user: Optional[User] = None): - """Checks for both upload *and* write permission""" - return self.check_permission(user, write=True, upload=True) - - def check_moderator(self, user: Optional[User]): - return self.check_permission(user, moderator=True) - - def check_admin(self, user: Optional[User]): - return self.check_permission(user, admin=True) - def messages_size(self): """Returns the number and total size (in bytes) of non-deleted messages currently stored in this room. Size is reflects the size of uploaded message bodies, not necessarily the size @@ -614,7 +510,7 @@ def get_messages_for( padding-trimmed value actually stored in the database). """ - mod = self.check_moderator(user) + mod = self.default_bot.check_moderator(self, user) msgs = [] opt_count = sum(arg is not None for arg in (sequence, after, before, single)) + bool(recent) @@ -745,190 +641,6 @@ def get_messages_for( return msgs - def filtering(self): - settings = { - 'profanity_filter': config.PROFANITY_FILTER, - 'profanity_silent': config.PROFANITY_SILENT, - 'alphabet_filters': config.ALPHABET_FILTERS, - 'alphabet_silent': config.ALPHABET_SILENT, - } - if self.token in config.ROOM_OVERRIDES: - for k in ( - 'profanity_filter', - 'profanity_silent', - 'alphabet_filters', - 'alphabet_silent', - ): - if k in config.ROOM_OVERRIDES[self.token]: - settings[k] = config.ROOM_OVERRIDES[self.token][k] - return settings - - def filter_should_reply(self, filter_type, filter_lang): - """If the settings say we should reply to a filter, this returns a tuple of - - (profile name, message format, whisper) - - where profile name is the name we should use in the reply, message format is a string with - substitutions ready, and whisper is True/False depending on whether it should be whispered - to the user (True) or public (False). - - If we shouldn't reply this returns (None, None, None) - """ - - if not config.FILTER_SETTINGS: - return (None, None, None) - - reply_format = None - profile_name = 'SOGS' - public = False - - # Precedences from least to most specific so that we load values from least specific first - # then overwrite them if we find a value in a more specific section - room_precedence = ('*', self.token) - filter_precedence = ('*', filter_type, filter_lang) if filter_lang else ('*', filter_type) - - for r in room_precedence: - s1 = config.FILTER_SETTINGS.get(r) - if s1 is None: - continue - for f in filter_precedence: - settings = s1.get(f) - if settings is None: - continue - - rf = settings.get('reply') - pn = settings.get('profile_name') - pb = settings.get('public') - if rf is not None: - reply_format = random.choice(rf) - if pn is not None: - profile_name = pn - if pb is not None: - public = pb - - return (reply_format, profile_name, public) - - def should_filter(self, user: User, data: bytes): - """ - Checks a message for disallowed alphabets and profanity (if the profanity - filter is enabled). - - - Returns None if this message passes (i.e. didn't trigger any filter, or is - being posted by an admin to whom the filters don't apply). - - - Returns a callback if the message fails but should be silently accepted. The callback - should be called (with no arguments) *after* the filtered message is inserted into the db. - - - Throws PostRejected if the message should be rejected (and rejection passed back to the - user). - """ - msg_ = None - - def msg(): - nonlocal msg_ - if msg_ is None: - msg_ = Post(raw=data) - return msg_ - - if not config.FILTER_MODS and self.check_moderator(user): - return None - - filter_type = None - filter_lang = None - - filt = self.filtering() - alphabets = filt['alphabet_filters'] - for lang, pattern in alphabet_filter_patterns: - if lang not in alphabets: - continue - - if not pattern.search(msg().text): - continue - - # Filter it! - filter_type, filter_lang = 'alphabet', lang - break - - if not filter_type and filt['profanity_filter']: - import better_profanity - - for part in (msg().text, msg().username): - if better_profanity.profanity.contains_profanity(part): - filter_type = 'profanity' - break - - if not filter_type: - return None - - silent = filt[filter_type + '_silent'] - - msg_fmt, prof_name, pub = self.filter_should_reply(filter_type, filter_lang) - if msg_fmt: - pbmsg = protobuf.Content() - body = msg_fmt.format( - profile_name=(user.session_id if msg().username is None else msg().username), - profile_at="@" + user.session_id, - room_name=self.name, - room_token=self.token, - ).encode() - pbmsg.dataMessage.body = body - pbmsg.dataMessage.timestamp = int(time.time() * 1000) - pbmsg.dataMessage.profile.displayName = prof_name - - # Add two bytes padding so that session doesn't get confused by a lack of padding - pbmsg = pbmsg.SerializeToString() + b'\x80\x00' - - # Make a fake signing key based on prof_name and the server privkey (so that different - # names use different keys; otherwise the bot names overwrite each other in Session - # clients when a later message has a new profile name). - global filter_privkeys - if prof_name in filter_privkeys: - signingkey = filter_privkeys[prof_name] - else: - signingkey = SigningKey( - blake2b( - prof_name.encode() + crypto.server_signkey.encode(), key=b'sogsfiltering' - ) - ) - filter_privkeys[prof_name] = signingkey - - sig = signingkey.sign(pbmsg).signature - server_fake_user = User( - session_id='15' + signingkey.verify_key.encode().hex(), autovivify=True, touch=False - ) - - def insert_reply(): - query( - """ - INSERT INTO messages - (room, "user", data, data_size, signature, whisper) - VALUES - (:r, :u, :data, :data_size, :signature, :whisper) - """, - r=self.id, - u=server_fake_user.id, - data=pbmsg[:-2], - data_size=len(pbmsg), - signature=sig, - whisper=None if pub else user.id, - ) - - if filt[filter_type + '_silent']: - # Defer the insertion until after the filtered row gets inserted - return insert_reply - else: - insert_reply() - - elif silent: - # No reply, so just return an empty callback - def noop(): - pass - - return noop - - # FIXME: can we send back some error code that makes Session not retry? - raise PostRejected(f"filtration rejected message ({filter_type})") - def _own_files(self, msg_id: int, files: List[int], user): """ Associated any of the given file ids with the given message id. Only files that are recent, @@ -959,110 +671,6 @@ def _own_files(self, msg_id: int, files: List[int], user): bind_expanding=['ids'], ) - def add_post( - self, - user: User, - data: bytes, - sig: bytes, - *, - whisper_to: Optional[Union[User, str]] = None, - whisper_mods: bool = False, - files: List[int] = [], - ): - """ - Adds a post to the room. The user must have write permissions. - - Raises BadPermission() if the user doesn't have posting permission; PostRejected() if the - post was rejected (such as subclass PostRateLimited() if the post was rejected for too - frequent posting). - - Returns the message details. - """ - if not self.check_write(user): - raise BadPermission() - - if data is None or sig is None or len(sig) != 64: - raise InvalidData() - - whisper_mods = bool(whisper_mods) - if (whisper_to or whisper_mods) and not self.check_moderator(user): - app.logger.warning(f"Cannot post a whisper to {self}: {user} is not a moderator") - raise BadPermission() - - if whisper_to and not isinstance(whisper_to, User): - whisper_to = User(session_id=whisper_to, autovivify=True, touch=False) - - filtered = self.should_filter(user, data) - - with db.transaction(): - if rate_limit_size and not self.check_admin(user): - since_limit = time.time() - rate_limit_interval - recent_count = query( - """ - SELECT COUNT(*) FROM messages - WHERE room = :r AND "user" = :u AND posted >= :since - """, - r=self.id, - u=user.id, - since=since_limit, - ).first()[0] - - if recent_count >= rate_limit_size: - raise PostRateLimited() - - data_size = len(data) - unpadded_data = utils.remove_session_message_padding(data) - - msg_id = db.insert_and_get_pk( - """ - INSERT INTO messages - (room, "user", data, data_size, signature, filtered, whisper, whisper_mods) - VALUES - (:r, :u, :data, :data_size, :signature, :filtered, :whisper, :whisper_mods) - """, - "id", - r=self.id, - u=user.id, - data=unpadded_data, - data_size=data_size, - signature=sig, - filtered=filtered is not None, - whisper=whisper_to.id if whisper_to else None, - whisper_mods=whisper_mods, - ) - - if files: - # Take ownership of any uploaded files attached to the post: - self._own_files(msg_id, files, user) - - assert msg_id is not None - row = query("SELECT posted, seqno FROM messages WHERE id = :m", m=msg_id).first() - msg = { - 'id': msg_id, - 'session_id': user.session_id, - 'posted': row[0], - 'seqno': row[1], - 'data': data, - 'signature': sig, - 'reactions': {}, - } - if filtered is not None: - msg['filtered'] = True - if whisper_to or whisper_mods: - msg['whisper'] = True - msg['whisper_mods'] = whisper_mods - if whisper_to: - msg['whisper_to'] = whisper_to.session_id - - # Don't call this inside the transaction because, if it's inserting a reply, we want the - # reply to have a later timestamp for proper ordering (because the timestamp inside a - # transaction is the time when the transaction started). - if filtered is not None: - filtered() - - send_mule("message_posted", msg['id']) - return msg - def edit_post(self, user: User, msg_id: int, data: bytes, sig: bytes, *, files: List[int] = []): """ Edits a post in the room. The post must exist, must have been authored by the same user, @@ -1078,7 +686,7 @@ def edit_post(self, user: User, msg_id: int, data: bytes, sig: bytes, *, files: profanity filter. - NoSuchPost() if the post is deleted. """ - if not self.check_write(user): + if not self.default_bot.check_write(self, user): raise BadPermission() if data is None or sig is None or len(sig) != 64: @@ -1161,7 +769,7 @@ def delete_posts(self, message_ids: List[int], deleter: User): ) if ids: - if not self.check_moderator(deleter): + if not self.default_bot.check_moderator(self, deleter): # If not a moderator then we only proceed if all of the messages are the # user's own: res = query( @@ -1196,9 +804,11 @@ def delete_all_posts(self, poster: User, *, deleter: User): """ fail = None - if poster.id != deleter.id and not self.check_moderator(deleter): + if poster.id != deleter.id and not self.default_bot.check_moderator(self, deleter): fail = "user is not a moderator" - elif self.check_admin(poster) and not self.check_admin(deleter): + elif self.default_bot.check_admin(self, poster) and not self.default_bot.check_admin( + self, deleter + ): fail = "only admins can delete all posts of another admin" if fail is not None: @@ -1370,7 +980,11 @@ def _check_reaction_request( if user_required and not user: app.logger.warning("Reaction request requires user authentication") raise BadPermission() - if not (self.check_moderator(user) if mod_required else self.check_read(user)): + if not ( + self.default_bot.check_moderator(self, user) + if mod_required + else self.default_bot.check_read(self, user) + ): app.logger.warning("Reaction request requires moderator authentication") raise BadPermission() @@ -1525,7 +1139,7 @@ def get_mods(self, user=None): ([public_mods], [public_admins], [hidden_mods], [hidden_admins]) """ - visible_clause = "" if self.check_moderator(user) else "AND visible_mod" + visible_clause = "" if self.default_bot.check_moderator(self, user) else "AND visible_mod" m, hm, a, ha = [], [], [], [] for session_id, visible, admin in query( f""" @@ -1580,7 +1194,7 @@ def set_moderator(self, user: User, *, added_by: User, admin=False, visible=True added_by is the user performing the update and must have admin permission. """ - if not self.check_admin(added_by): + if not self.default_bot.check_admin(self, added_by): app.logger.warning( f"Unable to set {user} as {'admin' if admin else 'moderator'} of {self}: " f"{added_by} is not an admin" @@ -1625,7 +1239,7 @@ def remove_moderator(self, user: User, *, removed_by: User, remove_admin_only: b a room moderator if already a room moderator or admin. """ - if not self.check_admin(removed_by): + if not self.default_bot.check_admin(self, removed_by): raise BadPermission() with db.transaction(): @@ -1664,13 +1278,15 @@ def ban_user(self, to_ban: User, *, mod: User, timeout: Optional[float] = None): with db.transaction(): with to_ban.check_blinding() as to_ban: fail = None - if not self.check_moderator(mod): + if not self.default_bot.check_moderator(self, mod): fail = "user is not a moderator" elif to_ban.id == mod.id: fail = "self-ban not permitted" elif to_ban.global_moderator: fail = "global mods/admins cannot be banned" - elif self.check_moderator(to_ban) and not self.check_admin(mod): + elif self.default_bot.check_moderator( + self, to_ban + ) and not self.default_bot.check_admin(self, mod): fail = "only admins can ban room mods/admins" if fail is not None: @@ -1724,7 +1340,7 @@ def unban_user(self, to_unban: User, *, mod: User): Throws on other errors (e.g. permission denied). """ - if not self.check_moderator(mod): + if not self.default_bot.check_moderator(self, mod): app.logger.warning(f"Error unbanning {to_unban} from {self} by {mod}: not a moderator") raise BadPermission() @@ -1795,7 +1411,7 @@ def set_permissions(self, user: User, *, mod: User, **perms): "Room.set_permissions: at least one of {', '.join(perm_types)} must be specified" ) - if not self.check_moderator(mod): + if not self.default_bot.check_moderator(self, mod): app.logger.warning(f"Error set perms {perms} on {user} by {mod}: not a moderator") raise BadPermission() @@ -1880,7 +1496,7 @@ def clear_future_permissions( def add_future_permission( self, - user, + user: User, *, at: float, mod: User, @@ -1955,7 +1571,7 @@ def upload_file( Returns the id of the newly inserted file row. Throws on error. """ - if not self.check_upload(uploader): + if not self.default_bot.check_upload(self, uploader): raise BadPermission() files_dir = os.path.join(config.UPLOAD_PATH, self.token) @@ -2047,7 +1663,7 @@ def pin(self, msg_id: int, admin: User): reorder pins, which are always sorted oldest-to-newest). """ - if not self.check_admin(admin): + if not self.default_bot.check_admin(self, admin): app.logger.warning(f"Unable to pin message to {self}: {admin} is not an admin") raise BadPermission() @@ -2074,7 +1690,7 @@ def unpin_all(self, admin: User): number of pinned messages removed. """ - if not self.check_admin(admin): + if not self.default_bot.check_admin(self, admin): app.logger.warning("Unable to unpin all messages from {self}: {admin} is not an admin") raise BadPermission() @@ -2106,7 +1722,7 @@ def unpin(self, msg_id: int, admin: User): pinned messages actually removed (i.e. 0 or 1). """ - if not self.check_admin(admin): + if not self.default_bot.check_admin(self, admin): app.logger.warning("Unable to unpin message from {self}: {admin} is not an admin") raise BadPermission() diff --git a/sogs/postfork.py b/sogs/postfork.py index 7c544c9d..6800e24b 100644 --- a/sogs/postfork.py +++ b/sogs/postfork.py @@ -11,7 +11,6 @@ def __init__(self, f): def __call__(self, f): pass - else: import uwsgidecorators diff --git a/sogs/routes/legacy.py b/sogs/routes/legacy.py index 1dee43b7..a03254fc 100644 --- a/sogs/routes/legacy.py +++ b/sogs/routes/legacy.py @@ -186,7 +186,6 @@ def legacy_transform_message(m): @legacy.post("/messages") def handle_post_legacy_message(): - user, room = legacy_check_user_room(write=True) req = request.json diff --git a/sogs/routes/messages.py b/sogs/routes/messages.py index e9054d8c..b6397da8 100644 --- a/sogs/routes/messages.py +++ b/sogs/routes/messages.py @@ -360,21 +360,15 @@ def post_message(room: Room): """ req = request.json - if room.bot_mode: - room.bot.receive_message( - user = g.user, - data = utils.decode_base64(req.get('data')), - sig = utils.decode_base64(req.get('signature')), - files = [int(x) for x in req.get('files', [])]) - else: - msg = room.add_post( - g.user, - data=utils.decode_base64(req.get('data')), - sig=utils.decode_base64(req.get('signature')), - whisper_to=req.get('whisper_to'), - whisper_mods=bool(req.get('whisper_mods')), - files=[int(x) for x in req.get('files', [])], - ) + msg = room.default_bot.receive_message( + user=g.user, + room=room, + data=utils.decode_base64(req.get('data')), + sig=utils.decode_base64(req.get('signature')), + whisper_to=req.get('whisper_to'), + whisper_mods=bool(req.get('whisper_mods')), + files=[int(x) for x in req.get('files', [])], + ) return utils.jsonify_with_base64(msg), http.CREATED diff --git a/sogs/routes/rooms.py b/sogs/routes/rooms.py index 4487a71b..de0348cf 100644 --- a/sogs/routes/rooms.py +++ b/sogs/routes/rooms.py @@ -446,7 +446,6 @@ def set_permissions(room, sid): with db.transaction(): with user.check_blinding() as u: - if req.get('unschedule') is not False and any( p in perms for p in ('read', 'write', 'upload') ): diff --git a/tests/test_auth.py b/tests/test_auth.py index 923c111b..316224d7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -379,7 +379,6 @@ def test_auth_batch(client, db): def test_auth_legacy(client, db, admin, user, room): - # Make a legacy auth token to make sure it works as expected first, but also to make sure it # gets ignored when we use X-SOGS-*. raw_token = sogs.utils.make_legacy_token(admin.session_id) diff --git a/tests/test_files.py b/tests/test_files.py index e304b703..1f264e7c 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -156,7 +156,6 @@ def test_no_file_crosspost(client, room, room2, user, global_admin): def _file_upload(client, room, user, *, unsafe=False, utf=False, filename): - url_post = f"/room/{room.token}/file" file_content = random(1024) filename_escaped = urllib.parse.quote(filename.encode('utf-8')) diff --git a/tests/test_onion_requests.py b/tests/test_onion_requests.py index dd4d2fa5..3d519197 100644 --- a/tests/test_onion_requests.py +++ b/tests/test_onion_requests.py @@ -138,7 +138,6 @@ def decrypt_reply(data, *, v, enc_type): def test_v3(room, client): - # Construct an onion request for /room/test-room req = {'method': 'GET', 'endpoint': '/room/test-room'} data = build_payload(req, v=3, enc_type="xchacha20") @@ -154,7 +153,6 @@ def test_v3(room, client): def test_v3_authenticated(room, mod, client): - # Construct an onion request for /room/test-room req = {'method': 'GET', 'endpoint': '/room/test-room'} req['headers'] = auth.x_sogs(mod.ed_key, crypto.server_pubkey, req['method'], req['endpoint']) diff --git a/tests/test_room_routes.py b/tests/test_room_routes.py index e5980829..d1afe603 100644 --- a/tests/test_room_routes.py +++ b/tests/test_room_routes.py @@ -10,7 +10,6 @@ def test_list(client, room, room2, user, user2, admin, mod, global_mod, global_admin): - room2.default_write = False room2.default_upload = False @@ -750,17 +749,13 @@ def deleted_entry(id, seqno): *(deleted_entry(i, s) for i, s in ((2, 11), (4, 12), (5, 13), (8, 14), (9, 15))), ] assert get_and_clean_since(10) == [ - *(deleted_entry(i, s) for i, s in ((2, 11), (4, 12), (5, 13), (8, 14), (9, 15))), + *(deleted_entry(i, s) for i, s in ((2, 11), (4, 12), (5, 13), (8, 14), (9, 15))) ] assert get_and_clean_since(11) == [ - *(deleted_entry(i, s) for i, s in ((4, 12), (5, 13), (8, 14), (9, 15))), - ] - assert get_and_clean_since(13) == [ - *(deleted_entry(i, s) for i, s in ((8, 14), (9, 15))), - ] - assert get_and_clean_since(14) == [ - *(deleted_entry(i, s) for i, s in ((9, 15),)), + *(deleted_entry(i, s) for i, s in ((4, 12), (5, 13), (8, 14), (9, 15))) ] + assert get_and_clean_since(13) == [*(deleted_entry(i, s) for i, s in ((8, 14), (9, 15)))] + assert get_and_clean_since(14) == [*(deleted_entry(i, s) for i, s in ((9, 15),))] assert get_and_clean_since(15) == [] @@ -934,7 +929,6 @@ def room_json(): def test_posting(client, room, user, user2, mod, global_mod): - url_post = "/room/test-room/message" d, s = (utils.encode_base64(x) for x in (b"post 1", pad64("sig 1"))) r = sogs_post(client, url_post, {"data": d, "signature": s}, user) @@ -957,7 +951,6 @@ def test_posting(client, room, user, user2, mod, global_mod): def test_whisper_to(client, room, user, user2, mod, global_mod): - url_post = "/room/test-room/message" d, s = (utils.encode_base64(x) for x in (b"whisper 1", pad64("sig 1"))) p = {"data": d, "signature": s, "whisper_to": user2.session_id} @@ -1005,7 +998,6 @@ def test_whisper_to(client, room, user, user2, mod, global_mod): def test_whisper_mods(client, room, user, user2, mod, global_mod, admin): - url_post = "/room/test-room/message" d, s = (utils.encode_base64(x) for x in (b"whisper 1", pad64("sig 1"))) p = {"data": d, "signature": s, "whisper_mods": True} @@ -1045,7 +1037,6 @@ def test_whisper_mods(client, room, user, user2, mod, global_mod, admin): def test_whisper_both(client, room, user, user2, mod, admin): - # A whisper aimed at both a user *and* all mods (e.g. a warning to a user) url_post = "/room/test-room/message" @@ -1138,7 +1129,6 @@ def test_whisper_both(client, room, user, user2, mod, admin): def test_edits(client, room, user, user2, mod, global_admin): - url_post = "/room/test-room/message" d, s = (utils.encode_base64(x) for x in (b"post 1", pad64("sig 1"))) r = sogs_post(client, url_post, {"data": d, "signature": s}, user) @@ -1401,7 +1391,6 @@ def test_set_room_perms(client, room, user, mod): def test_set_room_perm_futures(client, room, user, mod): - r = sogs_post( client, '/sequence', diff --git a/tests/test_rooms.py b/tests/test_rooms.py index 59b4469e..0f21311c 100644 --- a/tests/test_rooms.py +++ b/tests/test_rooms.py @@ -9,7 +9,6 @@ def test_create(room, room2): - r3 = Room.create('Test_Room-3', name='Test room 3', description='Test suite testing room3') rooms = get_rooms() @@ -36,7 +35,6 @@ def test_create(room, room2): def test_token_insensitive(room): - r = Room.create('Test_Ro-om', name='TR2', description='Test suite testing room2') r_a = Room(token='Test_Ro-om') @@ -92,7 +90,6 @@ def test_info(room): def test_updates(room): - assert room.message_sequence == 0 and room.info_updates == 0 and room.name == 'Test room' room.name = 'Test Room' @@ -118,7 +115,6 @@ def test_updates(room): def test_permissions(room, user, user2, mod, admin, global_mod, global_admin): - # Public permissions: assert not room.check_permission(admin=True) assert not room.check_permission(moderator=True) @@ -386,7 +382,6 @@ def test_bans(room, user, user2, mod, admin, global_mod, global_admin): def test_mods(room, user, user2, mod, admin, global_mod, global_admin): - room.set_moderator(user, added_by=admin) assert room.check_moderator(user) assert not room.check_admin(user) @@ -458,7 +453,6 @@ def test_mods(room, user, user2, mod, admin, global_mod, global_admin): def test_upload(room, user): - import os file = File(id=room.upload_file(content=b'abc', uploader=user, filename="abc.txt", lifetime=30)) @@ -489,7 +483,6 @@ def test_upload(room, user): def test_upload_expiry(room, user): - import os file = File(id=room.upload_file(content=b'abc', uploader=user, filename="abc.txt", lifetime=-1)) @@ -512,7 +505,6 @@ def test_upload_expiry(room, user): def test_image(room, user): - assert room.image is None fid = room.upload_file(content=b'abc', uploader=user, filename="abc.txt") diff --git a/tests/test_routes_general.py b/tests/test_routes_general.py index 9744b494..169d215c 100644 --- a/tests/test_routes_general.py +++ b/tests/test_routes_general.py @@ -134,7 +134,6 @@ def batch_test_endpoint4(): def test_batch(client): - d1, b1_exp = batch_data() b1 = client.post("/batch", json=d1) assert b1.json == b1_exp diff --git a/tests/test_user_routes.py b/tests/test_user_routes.py index 9fe54063..ea2fcb43 100644 --- a/tests/test_user_routes.py +++ b/tests/test_user_routes.py @@ -5,7 +5,6 @@ def test_global_mods(client, room, room2, user, user2, mod, admin, global_admin, global_mod): - assert not room2.check_moderator(user) assert not room2.check_moderator(user2) @@ -167,7 +166,6 @@ def test_global_mods(client, room, room2, user, user2, mod, admin, global_admin, def test_room_mods(client, room, room2, user, user2, mod, admin, global_admin, global_mod): - # Track expected info_updates values; the initial values are because creating the mod/admin/etc. # fixtures imported here perform db modifications that trigger updates (2 global mods + 2 mods # of `room`): From 2cd38c760faa49f6c901adceb63b08b8ab9a2b30 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Feb 2023 11:18:29 -0800 Subject: [PATCH 03/11] siloed mule and omq into classes --- contrib/uwsgi-sogs-standalone.ini | 2 +- sogs/model/bothandler.py | 240 ++++++++++++++++++++++++++++++ sogs/model/room.py | 13 +- sogs/mule.py | 137 +++++++++-------- sogs/omq.py | 73 +++++---- sogs/routes/messages.py | 19 +-- 6 files changed, 373 insertions(+), 111 deletions(-) create mode 100644 sogs/model/bothandler.py diff --git a/contrib/uwsgi-sogs-standalone.ini b/contrib/uwsgi-sogs-standalone.ini index 16a7f977..7a149a1b 100644 --- a/contrib/uwsgi-sogs-standalone.ini +++ b/contrib/uwsgi-sogs-standalone.ini @@ -27,7 +27,7 @@ processes = 2 enable-threads = true http = :80 mount = /=sogs.web:app -mule = sogs.mule:run +mule = @(call://sogs.mule.Mule.Mule) log-4xx = true log-5xx = true disable-logging = true diff --git a/sogs/model/bothandler.py b/sogs/model/bothandler.py new file mode 100644 index 00000000..35764c1a --- /dev/null +++ b/sogs/model/bothandler.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +from .. import crypto, db, config +from ..db import query +from ..web import app +from .exc import BadPermission, PostRateLimited +from .. import utils +from ..omq import send_mule +from .user import User +from .room import Room +from .message import Message +from .filter import SimpleFilter +from .exc import InvalidData + +from typing import Optional, List, Union +import time + +""" +Complications: + - given captcha bot + - new user (not approved) posts message + - we need bot to reply with whisper to that user with simple problem + - what does the bot do with the message they tried to send? + - can store locally + - user sends reply + - bot inserts it into room +""" + +""" +Control Flow: + 1) message comes in HTTP request + 2) unpacked/parsed/verified/permissions checked + 3) comes into relevant route (ex: add_post()) + 4) sends off to mule to be handled by bots + 5) mule has ordered list of bots by priority + 6) mule passes message to bots, which have fixed return values (insert, do not insert) + 7) if all bots approve, mule replies to worker with go ahead or vice versa for no go +""" + + +class BotHandler: + """ + Class representing an interface that manages active bots + + Object Properties: + bots - list of bots attached to room + """ + + def __init__( + self, + _rooms: List[Room], + row=None, + *, + id: Optional[int] = None, + session_id: Optional[int] = None, + ) -> None: + # immutable attributes + self.id = id + + def check_permission_for( + self, + room: Room, + user: Optional[User] = None, + *, + admin=False, + moderator=False, + read=False, + accessible=False, + write=False, + upload=False, + ): + """ + Checks whether `user` has the required permissions for this room and isn't banned. Returns + True if the user satisfies the permissions, False otherwise. If no user is provided then + permissions are checked against the room's defaults. + + Looked up permissions are cached within the Room instance so that looking up the same user + multiple times (i.e. from multiple parts of the code) does not re-query the database. + + Named arguments are as follows: + - admin -- if true then the user must have admin access to the room + - moderator -- if true then the user must have moderator (or admin) access to the room + - read -- if true then the user must have read access + - accessible -- if true then the user must have accessible access; note that this permission + is satisfied by *either* the `accessible` or `read` database flags (that is: read implies + accessible). + - write -- if true then the user must have write access + - upload -- if true then the user must have upload access; this should usually be combined + with write=True. + + You can specify multiple permissions as True, in which case all must be satisfied. If you + specify no permissions as required then the check only checks whether a user is banned but + otherwise requires no specific permission. + """ + + if user is None: + is_banned, can_read, can_access, can_write, can_upload, is_mod, is_admin = ( + False, + bool(room.default_read), + bool(room.default_accessible), + bool(room.default_write), + bool(room.default_upload), + False, + False, + ) + else: + if user.id not in self._perm_cache: + row = query( + """ + SELECT banned, read, accessible, write, upload, moderator, admin + FROM user_permissions + WHERE room = :r AND "user" = :u + """, + r=self.id, + u=user.id, + ).first() + self._perm_cache[user.id] = [bool(c) for c in row] + + ( + is_banned, + can_read, + can_access, + can_write, + can_upload, + is_mod, + is_admin, + ) = self._perm_cache[user.id] + + # Shortcuts for check_permission calls + def check_unbanned(self, room: Room, user: Optional[User]): + return self.check_permission_for(room, user) + + def check_read(self, room: Room, user: Optional[User] = None): + return self.check_permission_for(room, user, read=True) + + def check_accessible(self, room: Room, user: Optional[User] = None): + return self.check_permission_for(room, user, accessible=True) + + def check_write(self, room: Room, user: Optional[User] = None): + return self.check_permission_for(room, user, write=True) + + def check_upload(self, room: Room, user: Optional[User] = None): + """Checks for both upload *and* write permission""" + return self.check_permission_for(room, user, write=True, upload=True) + + def check_moderator(self, room: Room, user: Optional[User]): + return self.check_permission_for(room, user, moderator=True) + + def check_admin(self, room: Room, user: Optional[User]): + return self.check_permission_for(room, user, admin=True) + + def receive_message( + self, + room: Room, + user: User, + data: bytes, + sig: bytes, + *, + whisper_to: Optional[Union[User, str]] = None, + whisper_mods: bool = False, + files: List[int] = [], + ): + if not self.check_write(user): + raise BadPermission() + + if data is None or sig is None or len(sig) != 64: + raise InvalidData() + + whisper_mods = bool(whisper_mods) + if (whisper_to or whisper_mods) and not self.check_moderator(user): + app.logger.warning(f"Cannot post a whisper to {room}: {user} is not a moderator") + raise BadPermission() + + if whisper_to and not isinstance(whisper_to, User): + whisper_to = User(session_id=whisper_to, autovivify=True, touch=False) + + filtered = self.filter.read_message(user, data, room) + + with db.transaction(): + if room.rate_limit_size and not self.check_admin(user): + since_limit = time.time() - room.rate_limit_interval + recent_count = query( + """ + SELECT COUNT(*) FROM messages + WHERE room = :r AND "user" = :u AND posted >= :since + """, + r=self.id, + u=user.id, + since=since_limit, + ).first()[0] + + if recent_count >= room.rate_limit_size: + raise PostRateLimited() + + data_size = len(data) + unpadded_data = utils.remove_session_message_padding(data) + + msg_id = db.insert_and_get_pk( + """ + INSERT INTO messages + (room, "user", data, data_size, signature, filtered, whisper, whisper_mods) + VALUES + (:r, :u, :data, :data_size, :signature, :filtered, :whisper, :whisper_mods) + """, + "id", + r=room.id, + u=user.id, + data=unpadded_data, + data_size=data_size, + signature=sig, + filtered=filtered is not None, + whisper=whisper_to.id if whisper_to else None, + whisper_mods=whisper_mods, + ) + + if files: + # Take ownership of any uploaded files attached to the post: + room._own_files(msg_id, files, user) + + assert msg_id is not None + row = query("SELECT posted, seqno FROM messages WHERE id = :m", m=msg_id).first() + msg = { + 'id': msg_id, + 'session_id': user.session_id, + 'posted': row[0], + 'seqno': row[1], + 'data': data, + 'signature': sig, + 'reactions': {}, + } + if filtered is not None: + msg['filtered'] = True + if whisper_to or whisper_mods: + msg['whisper'] = True + msg['whisper_mods'] = whisper_mods + if whisper_to: + msg['whisper_to'] = whisper_to.session_id + + send_mule("message_posted", msg["id"]) + return msg diff --git a/sogs/model/room.py b/sogs/model/room.py index 8d829cc4..284f465d 100644 --- a/sogs/model/room.py +++ b/sogs/model/room.py @@ -79,18 +79,13 @@ def __init__(self, row=None, *, id=None, token=None, bot: Bot = None): looking up this raises a NoSuchRoom if no room with that token/id exists. """ self._refresh(id=id, token=token, row=row) - self.active_bot: bool = False - self.default_bot: Bot = bot if bot else Bot(_rooms=[self]) + self._active_bot: bool = False self.rate_limit_size = 5 self.rate_limit_interval = 16.0 - def activate_moderation(self): - self.default_bot.status[self] = True - self.active_bot = True - - def filter_only(self): - self.default_bot.status[self] = False - self.active_bot = False + @property.getter + def _bot_status(self) -> bool: + return self._active_bot def _refresh(self, *, id=None, token=None, row=None, perms=False): """ diff --git a/sogs/mule.py b/sogs/mule.py index f6777143..ab613e4d 100644 --- a/sogs/mule.py +++ b/sogs/mule.py @@ -8,96 +8,101 @@ from .web import app from . import cleanup from . import config -from . import omq as o +from .omq import OMQ +from .model.bothandler import BotHandler + # This is the uwsgi "mule" that handles things not related to serving HTTP requests: # - it holds the oxenmq instance (with its own interface into sogs) # - it handles cleanup jobs (e.g. periodic deletions) +def log_exceptions(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + try: + return f(*args, **kwargs) + except Exception as e: + app.logger.error(f"{f.__name__} raised exception: {e}") + raise -def run(): - try: - app.logger.info("OxenMQ mule started.") - - while True: - time.sleep(1) - - except Exception: - app.logger.error("mule died via exception:\n{}".format(traceback.format_exc())) - - -def allow_conn(addr, pk, sn): - # TODO: user recognition auth - return oxenmq.AuthLevel.basic + return wrapper +class Mule: -def admin_conn(addr, pk, sn): - return oxenmq.AuthLevel.admin + def __init__(self): + self.run() + def run(self): + try: + app.logger.info("OxenMQ mule started.") -def inproc_fail(connid, reason): - raise RuntimeError(f"Couldn't connect mule to itself: {reason}") + while True: + time.sleep(1) + except Exception: + app.logger.error("mule died via exception:\n{}".format(traceback.format_exc())) -def setup_omq(): - omq = o.omq + def allow_conn(self, addr, pk, sn): + # TODO: user recognition auth + return oxenmq.AuthLevel.basic - app.logger.debug("Mule setting up omq") - if isinstance(config.OMQ_LISTEN, list): - listen = config.OMQ_LISTEN - elif config.OMQ_LISTEN is None: - listen = [] - else: - listen = [config.OMQ_LISTEN] - for addr in listen: - omq.listen(addr, curve=True, allow_connection=allow_conn) - app.logger.info(f"OxenMQ listening on {addr}") + def admin_conn(self, addr, pk, sn): + return oxenmq.AuthLevel.admin - # Internal socket for workers to talk to us: - omq.listen(config.OMQ_INTERNAL, curve=False, allow_connection=admin_conn) + def inproc_fail(self, connid, reason): + raise RuntimeError(f"Couldn't connect mule to itself: {reason}") - # Periodic database cleanup timer: - omq.add_timer(cleanup.cleanup, timedelta(seconds=cleanup.INTERVAL)) + @log_exceptions + def message_posted(self, m: oxenmq.Message): + id = bt_deserialize(m.data()[0]) + app.logger.debug(f"FIXME: mule -- message posted stub, id={id}") - # Commands other workers can send to us, e.g. for notifications of activity for us to know about - worker = omq.add_category("worker", access_level=oxenmq.AuthLevel.admin) - worker.add_command("message_posted", message_posted) - worker.add_command("messages_deleted", messages_deleted) - worker.add_command("message_edited", message_edited) - app.logger.debug("Mule starting omq") - omq.start() + @log_exceptions + def messages_deleted(self, m: oxenmq.Message): + ids = bt_deserialize(m.data()[0]) + app.logger.debug(f"FIXME: mule -- message delete stub, deleted messages: {ids}") - # Connect mule to itself so that if something the mule does wants to send something to the mule - # it will work. (And so be careful not to recurse!) - app.logger.debug("Mule connecting to self") - o.mule_conn = omq.connect_inproc(on_success=None, on_failure=inproc_fail) + @log_exceptions + def message_edited(self, m: oxenmq.Message): + app.logger.debug("FIXME: mule -- message edited stub") -def log_exceptions(f): - @functools.wraps(f) - def wrapper(*args, **kwargs): - try: - return f(*args, **kwargs) - except Exception as e: - app.logger.error(f"{f.__name__} raised exception: {e}") - raise - return wrapper + def setup_omq(self, omq: OMQ): + app.logger.debug("Mule setting up omq") + if isinstance(config.OMQ_LISTEN, list): + listen = config.OMQ_LISTEN + elif config.OMQ_LISTEN is None: + listen = [] + else: + listen = [config.OMQ_LISTEN] + for addr in listen: + omq.listen(addr, curve=True, allow_connection=self.allow_conn) + app.logger.info(f"OxenMQ listening on {addr}") + # Internal socket for workers to talk to us: + omq._omq.listen(config.OMQ_INTERNAL, curve=False, allow_connection=self.admin_conn) -@log_exceptions -def message_posted(m: oxenmq.Message): - id = bt_deserialize(m.data()[0]) - app.logger.debug(f"FIXME: mule -- message posted stub, id={id}") + # Periodic database cleanup timer: + self._omq.add_timer(cleanup.cleanup, timedelta(seconds=cleanup.INTERVAL)) + # Commands other workers can send to us, e.g. for notifications of activity for us to know about + worker = self._omq.add_category("worker", access_level=oxenmq.AuthLevel.admin) + worker.add_command("message_posted", self.message_posted) + worker.add_command("messages_deleted", self.messages_deleted) + worker.add_command("message_edited", self.message_edited) -@log_exceptions -def messages_deleted(m: oxenmq.Message): - ids = bt_deserialize(m.data()[0]) - app.logger.debug(f"FIXME: mule -- message delete stub, deleted messages: {ids}") + ## NEW CODE FOR BOT + handler = self._omq.add_category("handler", access_level=oxenmq.AuthLevel.admin) + handler.add_command("add_bot", add_bot) + handler.add_command("remove_bot", remove_bot) + handler.add_command("send_to_handler", message_to_handler) + app.logger.debug("Mule starting omq") + self._omq.start() -@log_exceptions -def message_edited(m: oxenmq.Message): - app.logger.debug("FIXME: mule -- message edited stub") + # Connect mule to itself so that if something the mule does wants to send something to the mule + # it will work. (And so be careful not to recurse!) + app.logger.debug("Mule connecting to self") + omq.mule_conn = omq.connect_inproc(on_success=None, on_failure=inproc_fail) diff --git a/sogs/omq.py b/sogs/omq.py index 06bf2b7b..8535a09e 100644 --- a/sogs/omq.py +++ b/sogs/omq.py @@ -5,21 +5,55 @@ from oxenc import bt_serialize from . import crypto, config +from .mule import Mule from .postfork import postfork +from .model.bothandler import BotHandler -omq = None -mule_conn = None -test_suite = False +class OMQ: -def make_omq(): - omq = oxenmq.OxenMQ(privkey=crypto._privkey.encode(), pubkey=crypto.server_pubkey.encode()) + @postfork + def __init__(self): + try: + import uwsgi + except ModuleNotFoundError: + return + + self._omq = oxenmq.OxenMQ(privkey=crypto._privkey.encode(), pubkey=crypto.server_pubkey.encode()) + self._omq.ephemeral_routing_id = True - # We have multiple workers talking to the mule, so we *must* use ephemeral ids to not replace - # each others' connections. - omq.ephemeral_routing_id = True + self.bot_manager = BotHandler() + self.test_suite = False - return omq + if uwsgi.mule_id() != 0: + uwsgi.opt['mule'].setup_omq(self) + return + + from .web import app # Imported here to avoid circular import + + app.logger.debug(f"Starting oxenmq connection to mule in worker {uwsgi.worker_id()}") + + self._omq.start() + app.logger.debug("Started, connecting to mule") + self.mule_conn = self._omq.connect_remote(oxenmq.Address(config.OMQ_INTERNAL)) + + app.logger.debug(f"worker {uwsgi.worker_id()} connected to mule OMQ") + + + def send_mule(self, command, *args, prefix="worker."): + """ + Sends a command to the mule from a worker (or possibly from the mule itself). The command will + be prefixed with "worker." (unless overridden). + + Any args will be bt-serialized and send as message parts. + """ + if prefix: + command = prefix + command + + if self.test_suite and omq is None: + pass # TODO: for mule call testing we may want to do something else here? + else: + omq.send(mule_conn, command, *(bt_serialize(data) for data in args)) # Postfork for workers: we start oxenmq and connect to the mule process @@ -29,8 +63,11 @@ def start_oxenmq(): import uwsgi except ModuleNotFoundError: return + + + global omq, mule_conn, bot_manager - global omq, mule_conn + bot_manager = BotHandler() omq = make_omq() @@ -49,19 +86,3 @@ def start_oxenmq(): mule_conn = omq.connect_remote(oxenmq.Address(config.OMQ_INTERNAL)) app.logger.debug(f"worker {uwsgi.worker_id()} connected to mule OMQ") - - -def send_mule(command, *args, prefix="worker."): - """ - Sends a command to the mule from a worker (or possibly from the mule itself). The command will - be prefixed with "worker." (unless overridden). - - Any args will be bt-serialized and send as message parts. - """ - if prefix: - command = prefix + command - - if test_suite and omq is None: - pass # TODO: for mule call testing we may want to do something else here? - else: - omq.send(mule_conn, command, *(bt_serialize(data) for data in args)) diff --git a/sogs/routes/messages.py b/sogs/routes/messages.py index b6397da8..d5e84c13 100644 --- a/sogs/routes/messages.py +++ b/sogs/routes/messages.py @@ -1,6 +1,7 @@ from .. import http, utils from . import auth from model.room import Room +from ..omq import send_mule from flask import abort, jsonify, g, Blueprint, request @@ -360,15 +361,15 @@ def post_message(room: Room): """ req = request.json - msg = room.default_bot.receive_message( - user=g.user, - room=room, - data=utils.decode_base64(req.get('data')), - sig=utils.decode_base64(req.get('signature')), - whisper_to=req.get('whisper_to'), - whisper_mods=bool(req.get('whisper_mods')), - files=[int(x) for x in req.get('files', [])], - ) + send_mule(command="send_to_handler", + user=g.user, + room=room, + data=utils.decode_base64(req.get('data')), + sig=utils.decode_base64(req.get('signature')), + whisper_to=req.get('whisper_to'), + whisper_mods=bool(req.get('whisper_mods')), + files=[int(x) for x in req.get('files', [])], + prefix="handler.") return utils.jsonify_with_base64(msg), http.CREATED From cbaaecef762981e77552afe6312ad7d42e3ff48e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Feb 2023 15:57:02 -0800 Subject: [PATCH 04/11] squash this most likely --- contrib/uwsgi-sogs-standalone.ini | 2 +- sogs/model/{bothandler.py => manager.py} | 74 +++++++++++++++++++++--- sogs/mule.py | 16 +++-- sogs/omq.py | 62 +++++++------------- sogs/routes/messages.py | 22 +++---- sogs/utils.py | 2 + 6 files changed, 110 insertions(+), 68 deletions(-) rename sogs/model/{bothandler.py => manager.py} (81%) diff --git a/contrib/uwsgi-sogs-standalone.ini b/contrib/uwsgi-sogs-standalone.ini index 7a149a1b..6bd1eef2 100644 --- a/contrib/uwsgi-sogs-standalone.ini +++ b/contrib/uwsgi-sogs-standalone.ini @@ -27,7 +27,7 @@ processes = 2 enable-threads = true http = :80 mount = /=sogs.web:app -mule = @(call://sogs.mule.Mule.Mule) +mule = @(call://sogs.mule.Mule) log-4xx = true log-5xx = true disable-logging = true diff --git a/sogs/model/bothandler.py b/sogs/model/manager.py similarity index 81% rename from sogs/model/bothandler.py rename to sogs/model/manager.py index 35764c1a..ad8b21d3 100644 --- a/sogs/model/bothandler.py +++ b/sogs/model/manager.py @@ -7,12 +7,15 @@ from .. import utils from ..omq import send_mule from .user import User +from .bot import Bot from .room import Room from .message import Message from .filter import SimpleFilter from .exc import InvalidData +import heapq -from typing import Optional, List, Union +from dataclasses import dataclass, field +from typing import Optional, List, Union, Any import time """ @@ -23,10 +26,8 @@ - what does the bot do with the message they tried to send? - can store locally - user sends reply - - bot inserts it into room -""" + - bot inserts it into room (?) -""" Control Flow: 1) message comes in HTTP request 2) unpacked/parsed/verified/permissions checked @@ -38,12 +39,44 @@ """ -class BotHandler: +@dataclass(order=True) +class PriorityTuple(tuple): + priority: int + item: Any = field(compare=False) + + +# Simple "priority queue" of bots implemented using a dict with heap +# invariance maintained by qheap algorithm +# TODO: when bots are designed basically, add methods for polling them +# and receiving their judgments +class BotQueue: + def __init__(self) -> None: + self.queue = {} + + def _qsize(self) -> int: + return len(self.queue.keys()) + + def _empty(self) -> bool: + return not self._qsize() + + def _peek(self, priority: int): + return self.queue.get(priority) + + def _put(self, item: PriorityTuple): + temp = list(self.queue.items()) + heapq.heappush(temp, item) + self.queue = dict(temp) + + def _get(self): + return heapq.heappop(self.queue) + + +class Manager: """ Class representing an interface that manages active bots Object Properties: - bots - list of bots attached to room + queue - BotQueue object """ def __init__( @@ -54,8 +87,27 @@ def __init__( id: Optional[int] = None, session_id: Optional[int] = None, ) -> None: - # immutable attributes self.id = id + self.queue = BotQueue() + + def qempty(self): + return not self.queue._empty() + + def add_bot(self, bot: Bot, priority: int = None): + if not priority: + # if no priority is given, lowest priority is assigned + priority = self.qsize() + else: + # if priority is already taken, find next lowest + while self.queue.get(priority): + priority += 1 + self.queue._put(PriorityTuple(priority, bot)) + + def remove_bot(self): + do_something = 3 + + def peek(self, priority: int): + return self.queue._peek(priority) def check_permission_for( self, @@ -236,5 +288,13 @@ def receive_message( if whisper_to: msg['whisper_to'] = whisper_to.session_id + if room._bot_status(): + add_bot_logic = 3 + """ + TODO: add logic for bots receiving message and doing + bot things. The bots should be queried in terms of + priority, + """ + send_mule("message_posted", msg["id"]) return msg diff --git a/sogs/mule.py b/sogs/mule.py index ab613e4d..6f87543c 100644 --- a/sogs/mule.py +++ b/sogs/mule.py @@ -9,13 +9,14 @@ from . import cleanup from . import config from .omq import OMQ -from .model.bothandler import BotHandler +from .model.manager import Manager # This is the uwsgi "mule" that handles things not related to serving HTTP requests: # - it holds the oxenmq instance (with its own interface into sogs) # - it handles cleanup jobs (e.g. periodic deletions) + def log_exceptions(f): @functools.wraps(f) def wrapper(*args, **kwargs): @@ -27,8 +28,8 @@ def wrapper(*args, **kwargs): return wrapper -class Mule: +class Mule: def __init__(self): self.run() @@ -57,18 +58,15 @@ def message_posted(self, m: oxenmq.Message): id = bt_deserialize(m.data()[0]) app.logger.debug(f"FIXME: mule -- message posted stub, id={id}") - @log_exceptions def messages_deleted(self, m: oxenmq.Message): ids = bt_deserialize(m.data()[0]) app.logger.debug(f"FIXME: mule -- message delete stub, deleted messages: {ids}") - @log_exceptions def message_edited(self, m: oxenmq.Message): app.logger.debug("FIXME: mule -- message edited stub") - def setup_omq(self, omq: OMQ): app.logger.debug("Mule setting up omq") if isinstance(config.OMQ_LISTEN, list): @@ -95,9 +93,9 @@ def setup_omq(self, omq: OMQ): ## NEW CODE FOR BOT handler = self._omq.add_category("handler", access_level=oxenmq.AuthLevel.admin) - handler.add_command("add_bot", add_bot) - handler.add_command("remove_bot", remove_bot) - handler.add_command("send_to_handler", message_to_handler) + handler.add_command("add_bot", omq.add_bot) + handler.add_command("remove_bot", omq.remove_bot) + handler.add_command("send_to_handler", omq.manager.receive_message) app.logger.debug("Mule starting omq") self._omq.start() @@ -105,4 +103,4 @@ def setup_omq(self, omq: OMQ): # Connect mule to itself so that if something the mule does wants to send something to the mule # it will work. (And so be careful not to recurse!) app.logger.debug("Mule connecting to self") - omq.mule_conn = omq.connect_inproc(on_success=None, on_failure=inproc_fail) + omq.mule_conn = omq.connect_inproc(on_success=None, on_failure=self.inproc_fail) diff --git a/sogs/omq.py b/sogs/omq.py index 8535a09e..be378444 100644 --- a/sogs/omq.py +++ b/sogs/omq.py @@ -5,30 +5,31 @@ from oxenc import bt_serialize from . import crypto, config -from .mule import Mule from .postfork import postfork -from .model.bothandler import BotHandler +from .model.manager import Manager +omq_global = None class OMQ: - @postfork def __init__(self): try: import uwsgi except ModuleNotFoundError: return - - self._omq = oxenmq.OxenMQ(privkey=crypto._privkey.encode(), pubkey=crypto.server_pubkey.encode()) + + self._omq = oxenmq.OxenMQ( + privkey=crypto._privkey.encode(), pubkey=crypto.server_pubkey.encode() + ) self._omq.ephemeral_routing_id = True - self.bot_manager = BotHandler() + self.manager = Manager() self.test_suite = False if uwsgi.mule_id() != 0: uwsgi.opt['mule'].setup_omq(self) return - + from .web import app # Imported here to avoid circular import app.logger.debug(f"Starting oxenmq connection to mule in worker {uwsgi.worker_id()}") @@ -39,6 +40,16 @@ def __init__(self): app.logger.debug(f"worker {uwsgi.worker_id()} connected to mule OMQ") + global omq_global + omq_global = self + + def add_bot(self): + self.manager.add_bot() + # TODO: add omq logic + + def remove_bot(self): + self.manager.remove_bot() + # TODO: add omq logic def send_mule(self, command, *args, prefix="worker."): """ @@ -47,42 +58,11 @@ def send_mule(self, command, *args, prefix="worker."): Any args will be bt-serialized and send as message parts. """ + if prefix: command = prefix + command - if self.test_suite and omq is None: + if self.test_suite and self._omq is None: pass # TODO: for mule call testing we may want to do something else here? else: - omq.send(mule_conn, command, *(bt_serialize(data) for data in args)) - - -# Postfork for workers: we start oxenmq and connect to the mule process -@postfork -def start_oxenmq(): - try: - import uwsgi - except ModuleNotFoundError: - return - - - global omq, mule_conn, bot_manager - - bot_manager = BotHandler() - - omq = make_omq() - - if uwsgi.mule_id() != 0: - from . import mule - - mule.setup_omq() - return - - from .web import app # Imported here to avoid circular import - - app.logger.debug(f"Starting oxenmq connection to mule in worker {uwsgi.worker_id()}") - - omq.start() - app.logger.debug("Started, connecting to mule") - mule_conn = omq.connect_remote(oxenmq.Address(config.OMQ_INTERNAL)) - - app.logger.debug(f"worker {uwsgi.worker_id()} connected to mule OMQ") + self._omq.send(self.mule_conn, command, *(bt_serialize(data) for data in args)) diff --git a/sogs/routes/messages.py b/sogs/routes/messages.py index d5e84c13..9d37ad8a 100644 --- a/sogs/routes/messages.py +++ b/sogs/routes/messages.py @@ -1,7 +1,7 @@ from .. import http, utils from . import auth from model.room import Room -from ..omq import send_mule +from ..omq import omq_global from flask import abort, jsonify, g, Blueprint, request @@ -361,15 +361,17 @@ def post_message(room: Room): """ req = request.json - send_mule(command="send_to_handler", - user=g.user, - room=room, - data=utils.decode_base64(req.get('data')), - sig=utils.decode_base64(req.get('signature')), - whisper_to=req.get('whisper_to'), - whisper_mods=bool(req.get('whisper_mods')), - files=[int(x) for x in req.get('files', [])], - prefix="handler.") + msg = omq_global.send_mule( + command="send_to_handler", + user=g.user, + room=room, + data=utils.decode_base64(req.get('data')), + sig=utils.decode_base64(req.get('signature')), + whisper_to=req.get('whisper_to'), + whisper_mods=bool(req.get('whisper_mods')), + files=[int(x) for x in req.get('files', [])], + prefix="handler.", + ) return utils.jsonify_with_base64(msg), http.CREATED diff --git a/sogs/utils.py b/sogs/utils.py index bea930ac..0a788fa6 100644 --- a/sogs/utils.py +++ b/sogs/utils.py @@ -3,6 +3,8 @@ from . import http import base64 + + from flask import request, abort, Response import json from typing import Union, Tuple From d8836d5f67668b6ef0c47f8ba7758300a9388070 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Feb 2023 16:13:06 -0800 Subject: [PATCH 05/11] converted all send_mule calls to omq_global.send_mule calls --- sogs/model/bot.py | 4 ++-- sogs/model/manager.py | 4 ++-- sogs/model/room.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sogs/model/bot.py b/sogs/model/bot.py index e3637398..a86ddaaf 100644 --- a/sogs/model/bot.py +++ b/sogs/model/bot.py @@ -5,7 +5,7 @@ from ..web import app from .exc import BadPermission, PostRateLimited from .. import utils -from ..omq import send_mule +from ..omq import omq_global from .user import User from .room import Room from .message import Message @@ -263,5 +263,5 @@ def receive_message( if whisper_to: msg['whisper_to'] = whisper_to.session_id - send_mule("message_posted", msg["id"]) + omq_global.send_mule("message_posted", msg["id"]) return msg diff --git a/sogs/model/manager.py b/sogs/model/manager.py index ad8b21d3..c4520513 100644 --- a/sogs/model/manager.py +++ b/sogs/model/manager.py @@ -5,8 +5,8 @@ from ..web import app from .exc import BadPermission, PostRateLimited from .. import utils -from ..omq import send_mule from .user import User +from ..omq import omq_global from .bot import Bot from .room import Room from .message import Message @@ -296,5 +296,5 @@ def receive_message( priority, """ - send_mule("message_posted", msg["id"]) + omq_global.send_mule("message_posted", msg["id"]) return msg diff --git a/sogs/model/room.py b/sogs/model/room.py index 284f465d..c9ab4838 100644 --- a/sogs/model/room.py +++ b/sogs/model/room.py @@ -1,7 +1,7 @@ from .. import config, crypto, db, utils, session_pb2 as protobuf from ..db import query from ..hashing import blake2b -from ..omq import send_mule +from ..omq import omq_global from ..web import app from .user import User from .file import File @@ -727,7 +727,7 @@ def edit_post(self, user: User, msg_id: int, data: bytes, sig: bytes, *, files: # If the edit includes new attachments then own them: self._own_files(msg_id, files, user) - send_mule("message_edited", msg_id) + omq_global.send_mule("message_edited", msg_id) def delete_posts(self, message_ids: List[int], deleter: User): """ From 2df1ef539d75311992214b2a38af27897add0c5d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Mar 2023 11:02:29 -0800 Subject: [PATCH 06/11] prototype bot API with omq auth --- sogs/model/bot.py | 267 -------------------- sogs/model/{manager.py => clientmanager.py} | 38 ++- sogs/model/filter.py | 14 +- sogs/model/room.py | 1 - sogs/mule.py | 8 +- sogs/omq.py | 26 +- sogs/routes/clients.py | 154 +++++++++++ sogs/routes/omq_auth.py | 126 +++++++++ sogs/utils.py | 4 + 9 files changed, 333 insertions(+), 305 deletions(-) delete mode 100644 sogs/model/bot.py rename sogs/model/{manager.py => clientmanager.py} (93%) create mode 100644 sogs/routes/clients.py create mode 100644 sogs/routes/omq_auth.py diff --git a/sogs/model/bot.py b/sogs/model/bot.py deleted file mode 100644 index a86ddaaf..00000000 --- a/sogs/model/bot.py +++ /dev/null @@ -1,267 +0,0 @@ -from __future__ import annotations - -from .. import crypto, db, config -from ..db import query -from ..web import app -from .exc import BadPermission, PostRateLimited -from .. import utils -from ..omq import omq_global -from .user import User -from .room import Room -from .message import Message -from .filter import SimpleFilter -from .exc import InvalidData - -from typing import Optional, List, Union -import time - - -class Bot: - """ - Class representing a simple bot to manage open group server - - Object Properties: - id - database primary key for user row - status - bot active(T)/passive(F) moderation status (key = room, value = bool) - rooms - reference to room(s) in which bot is active - filter - reference to filter object paired with bot - perm_cache - dict storing user info/status (synced from all rooms patrolled) - session_id - hex encoded session_id of the bot - banned - default to false - global_admin - default to true for bot - global_moderator - default to true for bot - visible_mod - default to true for bot - """ - - def __init__( - self, - _rooms: List[Room], - row=None, - *, - id: Optional[int] = None, - session_id: Optional[int] = None, - ) -> None: - # immutable attributes - self._banned: bool = False - self._global_admin: bool = False - self._global_moderator: bool = False - self._visible_mod: bool = False - self.id = id - - # operational attributes - self.rooms: List[Room] = _rooms - self.status = {r: False for r in self.rooms} - self.filter = SimpleFilter(_bot=self) - self.perm_cache = {} - self.current_message: Message = None - self.nlp_model = None - self.language = 'English' - self.word_blacklist = ['placeholderlist', 'until', 'we', 'decide', 'naughty', 'words'] - - def __setattr__(self, __name: str, __value) -> None: - if __name in ['_banned', '_global_admin', '_global_moderator', '_visible_mod']: - raise AttributeError('Cannot modify bots') - else: - setattr(self, __name, __value) - - def __delattr__(self, __name: str) -> None: - if __name in ['_banned', '_global_admin', '_global_moderator', '_visible_mod']: - raise AttributeError('Cannot modify bots') - else: - delattr(self, __name) - - def _link_room(self, _room: Room): - self.rooms.append(_room) - self.filter.rooms.append(_room) - - def _unlink_room(self, _room: Room): - self.rooms.remove(_room) - - if not self.rooms: - delete_bot_function_goes_here = True - - def _refresh_cache(self): - for r in self.rooms: - self.perm_cache = self.perm_cache | r._perm_cache - - def check_permission_for( - self, - room: Room, - user: Optional[User] = None, - *, - admin=False, - moderator=False, - read=False, - accessible=False, - write=False, - upload=False, - ): - """ - Checks whether `user` has the required permissions for this room and isn't banned. Returns - True if the user satisfies the permissions, False otherwise. If no user is provided then - permissions are checked against the room's defaults. - - Looked up permissions are cached within the Room instance so that looking up the same user - multiple times (i.e. from multiple parts of the code) does not re-query the database. - - Named arguments are as follows: - - admin -- if true then the user must have admin access to the room - - moderator -- if true then the user must have moderator (or admin) access to the room - - read -- if true then the user must have read access - - accessible -- if true then the user must have accessible access; note that this permission - is satisfied by *either* the `accessible` or `read` database flags (that is: read implies - accessible). - - write -- if true then the user must have write access - - upload -- if true then the user must have upload access; this should usually be combined - with write=True. - - You can specify multiple permissions as True, in which case all must be satisfied. If you - specify no permissions as required then the check only checks whether a user is banned but - otherwise requires no specific permission. - """ - - if user is None: - is_banned, can_read, can_access, can_write, can_upload, is_mod, is_admin = ( - False, - bool(room.default_read), - bool(room.default_accessible), - bool(room.default_write), - bool(room.default_upload), - False, - False, - ) - else: - if user.id not in self._perm_cache: - row = query( - """ - SELECT banned, read, accessible, write, upload, moderator, admin - FROM user_permissions - WHERE room = :r AND "user" = :u - """, - r=self.id, - u=user.id, - ).first() - self._perm_cache[user.id] = [bool(c) for c in row] - - ( - is_banned, - can_read, - can_access, - can_write, - can_upload, - is_mod, - is_admin, - ) = self._perm_cache[user.id] - - # Shortcuts for check_permission calls - def check_unbanned(self, room: Room, user: Optional[User]): - return self.check_permission_for(room, user) - - def check_read(self, room: Room, user: Optional[User] = None): - return self.check_permission_for(room, user, read=True) - - def check_accessible(self, room: Room, user: Optional[User] = None): - return self.check_permission_for(room, user, accessible=True) - - def check_write(self, room: Room, user: Optional[User] = None): - return self.check_permission_for(room, user, write=True) - - def check_upload(self, room: Room, user: Optional[User] = None): - """Checks for both upload *and* write permission""" - return self.check_permission_for(room, user, write=True, upload=True) - - def check_moderator(self, room: Room, user: Optional[User]): - return self.check_permission_for(room, user, moderator=True) - - def check_admin(self, room: Room, user: Optional[User]): - return self.check_permission_for(room, user, admin=True) - - def receive_message( - self, - room: Room, - user: User, - data: bytes, - sig: bytes, - *, - whisper_to: Optional[Union[User, str]] = None, - whisper_mods: bool = False, - files: List[int] = [], - ): - if not self.check_write(user): - raise BadPermission() - - if data is None or sig is None or len(sig) != 64: - raise InvalidData() - - whisper_mods = bool(whisper_mods) - if (whisper_to or whisper_mods) and not self.check_moderator(user): - app.logger.warning(f"Cannot post a whisper to {room}: {user} is not a moderator") - raise BadPermission() - - if whisper_to and not isinstance(whisper_to, User): - whisper_to = User(session_id=whisper_to, autovivify=True, touch=False) - - filtered = self.filter.read_message(user, data, room) - - with db.transaction(): - if room.rate_limit_size and not self.check_admin(user): - since_limit = time.time() - room.rate_limit_interval - recent_count = query( - """ - SELECT COUNT(*) FROM messages - WHERE room = :r AND "user" = :u AND posted >= :since - """, - r=self.id, - u=user.id, - since=since_limit, - ).first()[0] - - if recent_count >= room.rate_limit_size: - raise PostRateLimited() - - data_size = len(data) - unpadded_data = utils.remove_session_message_padding(data) - - msg_id = db.insert_and_get_pk( - """ - INSERT INTO messages - (room, "user", data, data_size, signature, filtered, whisper, whisper_mods) - VALUES - (:r, :u, :data, :data_size, :signature, :filtered, :whisper, :whisper_mods) - """, - "id", - r=room.id, - u=user.id, - data=unpadded_data, - data_size=data_size, - signature=sig, - filtered=filtered is not None, - whisper=whisper_to.id if whisper_to else None, - whisper_mods=whisper_mods, - ) - - if files: - # Take ownership of any uploaded files attached to the post: - room._own_files(msg_id, files, user) - - assert msg_id is not None - row = query("SELECT posted, seqno FROM messages WHERE id = :m", m=msg_id).first() - msg = { - 'id': msg_id, - 'session_id': user.session_id, - 'posted': row[0], - 'seqno': row[1], - 'data': data, - 'signature': sig, - 'reactions': {}, - } - if filtered is not None: - msg['filtered'] = True - if whisper_to or whisper_mods: - msg['whisper'] = True - msg['whisper_mods'] = whisper_mods - if whisper_to: - msg['whisper_to'] = whisper_to.session_id - - omq_global.send_mule("message_posted", msg["id"]) - return msg diff --git a/sogs/model/manager.py b/sogs/model/clientmanager.py similarity index 93% rename from sogs/model/manager.py rename to sogs/model/clientmanager.py index c4520513..a8a3790e 100644 --- a/sogs/model/manager.py +++ b/sogs/model/clientmanager.py @@ -7,10 +7,7 @@ from .. import utils from .user import User from ..omq import omq_global -from .bot import Bot from .room import Room -from .message import Message -from .filter import SimpleFilter from .exc import InvalidData import heapq @@ -71,7 +68,7 @@ def _get(self): return heapq.heappop(self.queue) -class Manager: +class ClientManager: """ Class representing an interface that manages active bots @@ -88,12 +85,20 @@ def __init__( session_id: Optional[int] = None, ) -> None: self.id = id - self.queue = BotQueue() + self.filter = False + self.bqueue = BotQueue() + self.clients = [] - def qempty(self): - return not self.queue._empty() - def add_bot(self, bot: Bot, priority: int = None): + def bqempty(self): + return not self.bqueue._empty() + + + def register_client(self, cid, authlevel, bot: bool = False, priority: int = None): + if not bot: + # add client to self.clients + return + if not priority: # if no priority is given, lowest priority is assigned priority = self.qsize() @@ -101,13 +106,20 @@ def add_bot(self, bot: Bot, priority: int = None): # if priority is already taken, find next lowest while self.queue.get(priority): priority += 1 - self.queue._put(PriorityTuple(priority, bot)) + self.bqueue._put(PriorityTuple(priority, bot)) + - def remove_bot(self): - do_something = 3 + def deregister_client(self, cid, bot: bool = False): + if not bot: + # remove client from clients list + return + + # remove bot from bot queue + def peek(self, priority: int): - return self.queue._peek(priority) + return self.bqueue._peek(priority) + def check_permission_for( self, @@ -226,7 +238,7 @@ def receive_message( if whisper_to and not isinstance(whisper_to, User): whisper_to = User(session_id=whisper_to, autovivify=True, touch=False) - filtered = self.filter.read_message(user, data, room) + filtered = (self.filter)[False, self.filter.read_message(user, data, room)] with db.transaction(): if room.rate_limit_size and not self.check_admin(user): diff --git a/sogs/model/filter.py b/sogs/model/filter.py index edbdf320..60191587 100644 --- a/sogs/model/filter.py +++ b/sogs/model/filter.py @@ -1,22 +1,17 @@ from __future__ import annotations -from .. import config, crypto, db, utils, session_pb2 as protobuf +from .. import config, crypto, session_pb2 as protobuf import random -from .. import crypto, db, config +from .. import crypto, config from ..db import query from ..hashing import blake2b -from ..web import app from nacl.signing import SigningKey -from .exc import NoSuchUser, BadPermission, PostRejected +from .exc import PostRejected from sogs.model.user import User -from sogs.model.bot import Bot from sogs.model.room import Room, alphabet_filter_patterns from sogs.model.post import Post -from .exc import InvalidData -from typing import Optional, List, Union import time -import contextlib class SimpleFilter: @@ -28,8 +23,7 @@ class SimpleFilter: current_message - reference to current data being analyzed """ - def __init__(self, _bot: Bot, _room: Room = None): - self.bot: Bot = _bot + def __init__(self): self.current_message: Post = None def filtering(self): diff --git a/sogs/model/room.py b/sogs/model/room.py index c9ab4838..3000a943 100644 --- a/sogs/model/room.py +++ b/sogs/model/room.py @@ -154,7 +154,6 @@ def _refresh(self, *, id=None, token=None, row=None, perms=False): if perms or not hasattr(self, '_perm_cache'): self._perm_cache = {} - self.default_bot._refresh_cache() def __str__(self): """Returns `Room[token]` when converted to a str""" diff --git a/sogs/mule.py b/sogs/mule.py index 6f87543c..2d88f6ce 100644 --- a/sogs/mule.py +++ b/sogs/mule.py @@ -9,7 +9,7 @@ from . import cleanup from . import config from .omq import OMQ -from .model.manager import Manager +from .model.clientmanager import ClientManager # This is the uwsgi "mule" that handles things not related to serving HTTP requests: @@ -91,10 +91,10 @@ def setup_omq(self, omq: OMQ): worker.add_command("messages_deleted", self.messages_deleted) worker.add_command("message_edited", self.message_edited) - ## NEW CODE FOR BOT + # new client code handler = self._omq.add_category("handler", access_level=oxenmq.AuthLevel.admin) - handler.add_command("add_bot", omq.add_bot) - handler.add_command("remove_bot", omq.remove_bot) + handler.add_command("register_client", omq.register_client) + handler.add_command("deregister_client", omq.deregister_client) handler.add_command("send_to_handler", omq.manager.receive_message) app.logger.debug("Mule starting omq") diff --git a/sogs/omq.py b/sogs/omq.py index be378444..03eb047d 100644 --- a/sogs/omq.py +++ b/sogs/omq.py @@ -4,12 +4,15 @@ import oxenmq from oxenc import bt_serialize +from routes import omq_auth from . import crypto, config from .postfork import postfork -from .model.manager import Manager +from .model.clientmanager import ClientManager + omq_global = None + class OMQ: @postfork def __init__(self): @@ -23,7 +26,7 @@ def __init__(self): ) self._omq.ephemeral_routing_id = True - self.manager = Manager() + self.manager = ClientManager() self.test_suite = False if uwsgi.mule_id() != 0: @@ -32,25 +35,28 @@ def __init__(self): from .web import app # Imported here to avoid circular import - app.logger.debug(f"Starting oxenmq connection to mule in worker {uwsgi.worker_id()}") - + app.logger.debug(f"Starting oxenmq connection to mule in worker {uwsgi.worker_id()}...") self._omq.start() - app.logger.debug("Started, connecting to mule") + + app.logger.debug("Started, connecting to mule...") self.mule_conn = self._omq.connect_remote(oxenmq.Address(config.OMQ_INTERNAL)) - app.logger.debug(f"worker {uwsgi.worker_id()} connected to mule OMQ") + app.logger.debug(f"OMQ worker {uwsgi.worker_id()} connected to mule") global omq_global omq_global = self - def add_bot(self): - self.manager.add_bot() + + def register_client(self, cid, authlevel, bot: bool = False, priority: int = None): + self.manager.register_client(cid, authlevel, bot, priority) # TODO: add omq logic - def remove_bot(self): - self.manager.remove_bot() + + def deregister_client(self, cid, bot: bool = False): + self.manager.register_client() # TODO: add omq logic + def send_mule(self, command, *args, prefix="worker."): """ Sends a command to the mule from a worker (or possibly from the mule itself). The command will diff --git a/sogs/routes/clients.py b/sogs/routes/clients.py new file mode 100644 index 00000000..81d778ef --- /dev/null +++ b/sogs/routes/clients.py @@ -0,0 +1,154 @@ +from .. import db, http, utils +from ..model import room as mroom +from ..model.user import User +from ..web import app +from ..omq import omq_global +from . import omq_auth + +from flask import abort, jsonify, g, Blueprint, request + +# User-related routes + + +clients = Blueprint('clients', __name__) + + +@omq_auth.first_request +def register(cid): + """ + Registers a client with SOGS OMQ instance. In this context, "client" refers to any entity + seeking to create an authenticated OMQ connection. This may be, but is not limited to, + a user or a bot. + + ## URL Parameters + + - 'cid': the client ID of the given client to be registered with the SOGS instance + + ## Query Parameters + + - 'bot' (bool) : is bot or not + + ## Body Parameters + + Takes a JSON object as body with the following keys: + + - 'authlevel' : the oxenmq Authlevel to be attributed to the given client + - 'priority' : the priority level to be assigned to the given bot. If not passed, will be assigned + and handled by bot priority-queue + """ + + req = request.json + bot = utils.get_int_param('bot') # will set bot == 1 if key "bot" has value True + authlevel = req.get('authlevel') + priority = req.get('priority') + + client = (bot is 1)[register_client(cid, authlevel), + register_bot(cid, authlevel, priority)] + + return client + + +@clients.post("/client/registered/bot/") +def register_bot(cid, authlevel, priority): + """ + Registers a bot with SOGS OMQ instance + + ## URL Parameters + + - 'cid': the client ID of the given client to be registered with the SOGS instance + + ## Body Parameters + + Takes a JSON object as body with the following keys (passed as parameters from register()): + + - 'authlevel' : the oxenmq Authlevel to be attributed to the given client + - 'priority' : the priority level to be assigned to the given bot. If not passed, will be assigned + and handled by bot priority-queue + """ + + client = omq_global.send_mule( + command="register_client", + cid=cid, + authlevel=authlevel, + bot=True, + priority=priority + ) + + return client + + +@clients.post("/client/registered/client/") +def register_client(cid, authlevel): + """ + Registers a non-bot client with SOGS OMQ instance. In this context, "client" refers to any entity + seeking to create an authenticated OMQ connection. This may be, but is not limited to, + a user or a bot. + + ## URL Parameters + + - 'cid': the client ID of the given client to be registered with the SOGS instance + + ## Body Parameters + + Takes a JSON object as body with the following keys (passed as parameters from register()): + + - 'authlevel' : the oxenmq Authlevel to be attributed to the given client + """ + + client = omq_global.send_mule( + command="register_client", + cid=cid, + authlevel=authlevel, + bot=False, + priority=None + ) + + return client + + +@omq_auth.admin_required +def unregister(cid): + """ + Unegisters a non-bot client with SOGS OMQ instance + + ## URL Parameters + + - 'cid': the client ID of the given client to be registered with the SOGS instance + + ## Query Parameters + + - 'bot' (bool) : is bot or not + """ + + bot = utils.get_int_param('bot') + + client = (bot)[unregister_client(cid), unregister_bot(cid)] + + return client + + + +@clients.post("/client/deregistered/client/") +@clients.delete("/client/registered/client/") +def unregister_client(cid): + + client = omq_global.send_mule( + command="register_client", + cid=cid, + bot=False + ) + + return client + + +@clients.post("/bot/deregistered/bot/") +@clients.delete("/bot/registered/bot/") +def unregister_bot(cid): + + client = omq_global.send_mule( + command="register_client", + cid=cid, + bot=True + ) + + return client diff --git a/sogs/routes/omq_auth.py b/sogs/routes/omq_auth.py new file mode 100644 index 00000000..f83d9ba0 --- /dev/null +++ b/sogs/routes/omq_auth.py @@ -0,0 +1,126 @@ +import oxenmq +import auth +from ..web import app +from ..db import query +from .. import config, crypto, http, utils +from ..model.user import User +from ..hashing import blake2b + +from flask import request, abort, Response, g +import time +import nacl +from nacl.signing import VerifyKey +import nacl.exceptions +import nacl.bindings as sodium +import sqlalchemy.exc +from functools import wraps + +# Authentication for handling OMQ requests + + +def abort_request(code, msg, warn=True): + if warn: + app.logger.warning(msg) + else: + app.logger.debug(msg) + abort(Response(msg, status=code, mimetype='text/plain')) + + +def require_client(): + """ Requires that an authenticated client was found in the OMQ instance; aborts with + UNAUTHORIZED if the request has no client """ + if g.client_id is None: + abort_request(http.UNAUTHORIZED, 'OMQ client authentication required') + + +def client_required(f): + """ Decorator for an endpoint that requires a client; this calls require_client() at the + beginning of the request to abort the request as UNAUTHORIZED if the client has not been + previously authenticated""" + + @wraps(f) + def required_client_wrapper(*args, **kwargs): + require_client() + return f(*args, **kwargs) + + return required_client_wrapper + + +def require_authlevel(admin=True): + require_client() + if g.client_authlevel is not oxenmq.Authlevel.admin if admin else g.client_authlevel is not oxenmq.Authlevel.basic: + abort_request( + http.FORBIDDEN, + f"This endpoint requires oxenmq.Authlevel.{'admin' if admin else 'basic'} permissions" + ) + + +def basic_required(f): + """ Decorator for an endpoint that requires a client has basic OMQ authorization """ + + @wraps(f) + def required_basic_wrapper(*args, **kwargs): + require_authlevel(admin=False) + return f(*args, **kwargs) + + return required_basic_wrapper + + +def admin_required(f): + """ Decorator for an endpoint that requires a client has admin OMQ authorization """ + + @wraps(f) + def required_admin_wrapper(*args, **kwargs): + require_authlevel(admin=True) + return f(*args, **kwargs) + + return required_admin_wrapper + + +def first_request(f): + """ Decorator for an endpoint that will be the very first request for a given client. This + will ensure that the client is then registered for any subsequent requests. + + This function will typically take the folling parameters: + - cid : unique client ID to be attributed + - authlevel (oxenmq) + """ + + @wraps + def first_request_wrapper(*args, cid, authlevel, **kwargs): + handle_omq_registration(cid, authlevel) + return f(*args, cid=cid, authlevel=authlevel, **kwargs) + + return first_request_wrapper + + +def handle_omq_registration(sid, authlevel): + """ + Registers client with OMQ instance before its very first request + """ + if hasattr(g, 'client_id') and hasattr(g, 'client_authlevel') and not g.client_reauth: + app.logger.warning(f"Client {g.client_id} already registered for {g.client_authlevel} access") + return + + """ + Here goes ye olde OMQ registration logic. We need to decide what identification will + be used to verify every connected client s.t. that information persists for all subsequent + requests. + + In this registration, we need to set: + g.client_id + g.client_authlevel + """ + + +@app.before_request +def verify_omq_auth(): + """ + Verifies OMQ authentication before each request + """ + + # If there is already a g.o_id, then this is NOT the first request made by this client, unless + # g.client_reauth has been specifically set + if hasattr(g, 'client_id') and hasattr(g, 'client_authlevel') and not g.client_reauth: + app.logger.debug(f"Client {g.client_id} already authenticated for {g.client_authlevel} access") + return diff --git a/sogs/utils.py b/sogs/utils.py index 0a788fa6..9dbaf38e 100644 --- a/sogs/utils.py +++ b/sogs/utils.py @@ -136,6 +136,10 @@ def get_int_param(name, default=None, *, required=False, min=None, max=None, tru return default try: + if (val is "true" or val is "True" or val is True): + val = 1 + if (val is "false" or val is "False" or val is False): + val = 0 val = int(val) except Exception: abort(http.BAD_REQUEST) From 689c2d1411f8a639b0433c55db2d85414aed080b Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Mar 2023 07:40:31 -0700 Subject: [PATCH 07/11] in progress --- sogs/model/clientmanager.py | 4 +--- sogs/mule.py | 15 +++++++++++++++ sogs/routes/clients.py | 13 ++++++++----- sogs/routes/messages.py | 5 +++++ sogs/routes/omq_auth.py | 15 ++++++++++++++- 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/sogs/model/clientmanager.py b/sogs/model/clientmanager.py index a8a3790e..b681d109 100644 --- a/sogs/model/clientmanager.py +++ b/sogs/model/clientmanager.py @@ -15,6 +15,7 @@ from typing import Optional, List, Union, Any import time + """ Complications: - given captcha bot @@ -78,11 +79,8 @@ class ClientManager: def __init__( self, - _rooms: List[Room], - row=None, *, id: Optional[int] = None, - session_id: Optional[int] = None, ) -> None: self.id = id self.filter = False diff --git a/sogs/mule.py b/sogs/mule.py index 2d88f6ce..da4d3986 100644 --- a/sogs/mule.py +++ b/sogs/mule.py @@ -92,6 +92,7 @@ def setup_omq(self, omq: OMQ): worker.add_command("message_edited", self.message_edited) # new client code + # TOFIX: use add_request_command to handle a response value handler = self._omq.add_category("handler", access_level=oxenmq.AuthLevel.admin) handler.add_command("register_client", omq.register_client) handler.add_command("deregister_client", omq.deregister_client) @@ -104,3 +105,17 @@ def setup_omq(self, omq: OMQ): # it will work. (And so be careful not to recurse!) app.logger.debug("Mule connecting to self") omq.mule_conn = omq.connect_inproc(on_success=None, on_failure=self.inproc_fail) + + + +""" +TOFIX: oxenmq calls should pass as oxenmq.Message not as JSON + +def _on_subscribe(self, msg: oxenmq.Message): + code, message = None, None # If still None at the end, we send a reply + try: + args = json.loads(msg.data()[0]) + + pubkey = extract_hex_or_b64(args, "pubkey", 33) + +""" diff --git a/sogs/routes/clients.py b/sogs/routes/clients.py index 81d778ef..8100d685 100644 --- a/sogs/routes/clients.py +++ b/sogs/routes/clients.py @@ -9,6 +9,12 @@ # User-related routes +""" + TOFIX: + remove HTTP shit from requests + +""" + clients = Blueprint('clients', __name__) @@ -18,7 +24,7 @@ def register(cid): """ Registers a client with SOGS OMQ instance. In this context, "client" refers to any entity seeking to create an authenticated OMQ connection. This may be, but is not limited to, - a user or a bot. + a user or a bot ## URL Parameters @@ -80,9 +86,7 @@ def register_bot(cid, authlevel, priority): @clients.post("/client/registered/client/") def register_client(cid, authlevel): """ - Registers a non-bot client with SOGS OMQ instance. In this context, "client" refers to any entity - seeking to create an authenticated OMQ connection. This may be, but is not limited to, - a user or a bot. + Registers a non-bot client with SOGS OMQ instance ## URL Parameters @@ -127,7 +131,6 @@ def unregister(cid): return client - @clients.post("/client/deregistered/client/") @clients.delete("/client/registered/client/") def unregister_client(cid): diff --git a/sogs/routes/messages.py b/sogs/routes/messages.py index 9d37ad8a..e319f7a7 100644 --- a/sogs/routes/messages.py +++ b/sogs/routes/messages.py @@ -310,6 +310,11 @@ def message_single(room: Room, msg_id): return utils.jsonify_with_base64(msgs[0]) +""" + TOFIX: + - add some decorator to this s.t. it routes it to the correct OMQ endpoint +""" + @messages.post("/room//message") @auth.user_required def post_message(room: Room): diff --git a/sogs/routes/omq_auth.py b/sogs/routes/omq_auth.py index f83d9ba0..810e0fc2 100644 --- a/sogs/routes/omq_auth.py +++ b/sogs/routes/omq_auth.py @@ -105,7 +105,7 @@ def handle_omq_registration(sid, authlevel): """ Here goes ye olde OMQ registration logic. We need to decide what identification will be used to verify every connected client s.t. that information persists for all subsequent - requests. + requests In this registration, we need to set: g.client_id @@ -124,3 +124,16 @@ def verify_omq_auth(): if hasattr(g, 'client_id') and hasattr(g, 'client_authlevel') and not g.client_reauth: app.logger.debug(f"Client {g.client_id} already authenticated for {g.client_authlevel} access") return + + + +""" + TOFIX: + - add some type of dict in omq_global to map conn_ID (onenmq conn ID) to session_ID/other info + - do not persist: + - room specific access: check every time it makes a request because it can change + - values that admin level can change + + + +""" From dad5560dab5fadd636944e62c40ef8e86c062b3f Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 17 Mar 2023 12:27:59 -0700 Subject: [PATCH 08/11] large-ish commit. added map in global omq instance to track client connections, implementation of OMQ request handling underway, re-writing subrequest and onion-request handling to process omq requests --- sogs/model/clientmanager.py | 6 +- sogs/mule.py | 18 +++-- sogs/omq.py | 56 ++++++++++++--- sogs/routes/clients.py | 36 +++++----- sogs/routes/messages.py | 5 +- sogs/routes/omq_auth.py | 43 +++++++++--- sogs/routes/omq_subrequest.py | 64 +++++++++++++++++ sogs/routes/onion_omq_request.py | 113 +++++++++++++++++++++++++++++++ sogs/routes/rooms.py | 2 + sogs/routes/subrequest.py | 2 +- sogs/routes/users.py | 2 + sogs/routes/views.py | 2 + sogs/utils.py | 1 - 13 files changed, 302 insertions(+), 48 deletions(-) create mode 100644 sogs/routes/omq_subrequest.py create mode 100644 sogs/routes/onion_omq_request.py diff --git a/sogs/model/clientmanager.py b/sogs/model/clientmanager.py index b681d109..5ec60016 100644 --- a/sogs/model/clientmanager.py +++ b/sogs/model/clientmanager.py @@ -1,5 +1,7 @@ from __future__ import annotations +import oxenmq + from .. import crypto, db, config from ..db import query from ..web import app @@ -92,7 +94,7 @@ def bqempty(self): return not self.bqueue._empty() - def register_client(self, cid, authlevel, bot: bool = False, priority: int = None): + def register_client(self, conn_id, cid, authlevel, bot, priority): if not bot: # add client to self.clients return @@ -107,7 +109,7 @@ def register_client(self, cid, authlevel, bot: bool = False, priority: int = Non self.bqueue._put(PriorityTuple(priority, bot)) - def deregister_client(self, cid, bot: bool = False): + def deregister_client(self, cid, bot): if not bot: # remove client from clients list return diff --git a/sogs/mule.py b/sogs/mule.py index da4d3986..265c3b9e 100644 --- a/sogs/mule.py +++ b/sogs/mule.py @@ -86,17 +86,21 @@ def setup_omq(self, omq: OMQ): self._omq.add_timer(cleanup.cleanup, timedelta(seconds=cleanup.INTERVAL)) # Commands other workers can send to us, e.g. for notifications of activity for us to know about - worker = self._omq.add_category("worker", access_level=oxenmq.AuthLevel.admin) + worker = omq._omq.add_category("worker", access_level=oxenmq.AuthLevel.admin) worker.add_command("message_posted", self.message_posted) worker.add_command("messages_deleted", self.messages_deleted) worker.add_command("message_edited", self.message_edited) - # new client code - # TOFIX: use add_request_command to handle a response value - handler = self._omq.add_category("handler", access_level=oxenmq.AuthLevel.admin) - handler.add_command("register_client", omq.register_client) - handler.add_command("deregister_client", omq.deregister_client) - handler.add_command("send_to_handler", omq.manager.receive_message) + # client code + handler = omq._omq.add_category("handler", access_level=oxenmq.AuthLevel.admin) + handler.add_request_command("register_client", omq.register_client) + handler.add_request_command("deregister_client", omq.deregister_client) + handler.add_request_command("send_to_handler", omq.manager.receive_message) + + # proxy handler for subrequest queue + internal = omq._omq.add_category("internal", access_level=oxenmq.AuthLevel.admin) + internal.add_request_command("get_next_request", omq.get_next_request) + internal.add_request_command("subreq_response", omq.subreq_response) app.logger.debug("Mule starting omq") self._omq.start() diff --git a/sogs/omq.py b/sogs/omq.py index 03eb047d..df5a86f4 100644 --- a/sogs/omq.py +++ b/sogs/omq.py @@ -1,9 +1,10 @@ # Common oxenmq object; this is used by workers and the oxenmq mule. We create, but do not start, # this pre-forking. -import oxenmq -from oxenc import bt_serialize +import oxenmq, queue +from oxenc import bt_serialize, bt_deserialize +from mule import log_exceptions from routes import omq_auth from . import crypto, config from .postfork import postfork @@ -11,6 +12,8 @@ omq_global = None +global blueprints_global +blueprints_global = {} class OMQ: @@ -25,13 +28,16 @@ def __init__(self): privkey=crypto._privkey.encode(), pubkey=crypto.server_pubkey.encode() ) self._omq.ephemeral_routing_id = True - + self.client_map = {} self.manager = ClientManager() self.test_suite = False + self.subreq_queue = queue.SimpleQueue() if uwsgi.mule_id() != 0: uwsgi.opt['mule'].setup_omq(self) return + + uwsgi.register_signal(123, 'internal', self.handle_proxied_omq_req) from .web import app # Imported here to avoid circular import @@ -46,15 +52,47 @@ def __init__(self): global omq_global omq_global = self + + @log_exceptions + def subreq_response(self): + pass + + + @log_exceptions + def handle_proxied_omq_req(self): + id, subreq_body = self.send_mule( + command='get_next_request', + prefix='internal' + ) + + ''' + + Handle omq subrequest + + ''' + + return + + @log_exceptions + def get_next_request(self): + subreq_body = self.subreq_queue.get() + id = list(subreq_body.keys())[0] + return id, subreq_body[id] + - def register_client(self, cid, authlevel, bot: bool = False, priority: int = None): - self.manager.register_client(cid, authlevel, bot, priority) - # TODO: add omq logic + @log_exceptions + def register_client(self, msg: oxenmq.Message): + cid, authlevel, bot, priority = bt_deserialize(msg.data()[0]) + conn_id = msg.conn() + self.client_map[conn_id] = cid + self.manager.register_client(msg) - def deregister_client(self, cid, bot: bool = False): - self.manager.register_client() - # TODO: add omq logic + @log_exceptions + def deregister_client(self, msg: oxenmq.Message): + cid, bot = bt_deserialize(msg.data()[0]) + self.client_map.pop(cid) + self.manager.deregister_client(cid, bot) def send_mule(self, command, *args, prefix="worker."): diff --git a/sogs/routes/clients.py b/sogs/routes/clients.py index 8100d685..daf4bd23 100644 --- a/sogs/routes/clients.py +++ b/sogs/routes/clients.py @@ -2,7 +2,7 @@ from ..model import room as mroom from ..model.user import User from ..web import app -from ..omq import omq_global +from ..omq import omq_global, blueprints_global from . import omq_auth from flask import abort, jsonify, g, Blueprint, request @@ -15,8 +15,8 @@ """ - clients = Blueprint('clients', __name__) +blueprints_global['clients'] = clients @omq_auth.first_request @@ -28,7 +28,7 @@ def register(cid): ## URL Parameters - - 'cid': the client ID of the given client to be registered with the SOGS instance + - 'cid': the client ID (session ID) of the given client to be registered with the SOGS instance ## Query Parameters @@ -61,7 +61,7 @@ def register_bot(cid, authlevel, priority): ## URL Parameters - - 'cid': the client ID of the given client to be registered with the SOGS instance + - 'cid': the client ID (session ID) of the given client to be registered with the SOGS instance ## Body Parameters @@ -73,11 +73,12 @@ def register_bot(cid, authlevel, priority): """ client = omq_global.send_mule( - command="register_client", + command='register_client', cid=cid, authlevel=authlevel, - bot=True, - priority=priority + bot=1, + priority=priority, + prefix='handler' ) return client @@ -90,7 +91,7 @@ def register_client(cid, authlevel): ## URL Parameters - - 'cid': the client ID of the given client to be registered with the SOGS instance + - 'cid': the client ID (session ID) of the given client to be registered with the SOGS instance ## Body Parameters @@ -100,11 +101,12 @@ def register_client(cid, authlevel): """ client = omq_global.send_mule( - command="register_client", + command='register_client', cid=cid, authlevel=authlevel, - bot=False, - priority=None + bot=0, + priority=None, + prefix='handler' ) return client @@ -117,7 +119,7 @@ def unregister(cid): ## URL Parameters - - 'cid': the client ID of the given client to be registered with the SOGS instance + - 'cid': the client ID (session ID) of the given client to be registered with the SOGS instance ## Query Parameters @@ -136,9 +138,10 @@ def unregister(cid): def unregister_client(cid): client = omq_global.send_mule( - command="register_client", + command='unregister_client', cid=cid, - bot=False + bot=0, + prefix='handler' ) return client @@ -149,9 +152,10 @@ def unregister_client(cid): def unregister_bot(cid): client = omq_global.send_mule( - command="register_client", + command='unregister_bot', cid=cid, - bot=True + bot=1, + prefix='handler' ) return client diff --git a/sogs/routes/messages.py b/sogs/routes/messages.py index e319f7a7..37bb41e0 100644 --- a/sogs/routes/messages.py +++ b/sogs/routes/messages.py @@ -1,15 +1,14 @@ from .. import http, utils from . import auth from model.room import Room -from ..omq import omq_global +from ..omq import omq_global, blueprints_global from flask import abort, jsonify, g, Blueprint, request # Room message retrieving/submitting endpoints - messages = Blueprint('messages', __name__) - +blueprints_global['messages'] = messages def qs_reactors(): return utils.get_int_param('reactors', 4, min=0, max=20, truncate=True) diff --git a/sogs/routes/omq_auth.py b/sogs/routes/omq_auth.py index 810e0fc2..36b446de 100644 --- a/sogs/routes/omq_auth.py +++ b/sogs/routes/omq_auth.py @@ -3,10 +3,11 @@ from ..web import app from ..db import query from .. import config, crypto, http, utils +from ..omq import omq_global, blueprints_global from ..model.user import User from ..hashing import blake2b -from flask import request, abort, Response, g +from flask import request, abort, Response, g, Blueprint import time import nacl from nacl.signing import VerifyKey @@ -17,6 +18,30 @@ # Authentication for handling OMQ requests +omq = Blueprint('endpoint', __name__) + +def endpoint(f): + """ + Default endpoint for omq routes to pass requests to; constructs flask HTTP request + + Message (request) components: + + "blueprint" - the flask blueprint + "query" - the request query + "pubkey" - pk of client making request + "params" - a json value to dump as the the query parameters + + Example: + full request: `@omq.endpoint("messages", "room..messages.since.", {'Room:room', 'int:seqno'})` + blueprint: 'messages' + query: 'room..messages.since.' + params: {'Room:room', 'int:seqno'} + """ + + @wraps(f) + def endpoint_wrapper(*args, blueprint, query, pubkey, params, **kwargs): + bp = blueprints_global['messages'] + def abort_request(code, msg, warn=True): if warn: @@ -27,16 +52,20 @@ def abort_request(code, msg, warn=True): def require_client(): - """ Requires that an authenticated client was found in the OMQ instance; aborts with - UNAUTHORIZED if the request has no client """ + """ + Requires that an authenticated client was found in the OMQ instance; aborts with + UNAUTHORIZED if the request has no client + """ if g.client_id is None: abort_request(http.UNAUTHORIZED, 'OMQ client authentication required') def client_required(f): - """ Decorator for an endpoint that requires a client; this calls require_client() at the + """ + Decorator for an endpoint that requires a client; this calls require_client() at the beginning of the request to abort the request as UNAUTHORIZED if the client has not been - previously authenticated""" + previously authenticated + """ @wraps(f) def required_client_wrapper(*args, **kwargs): @@ -126,14 +155,10 @@ def verify_omq_auth(): return - """ TOFIX: - add some type of dict in omq_global to map conn_ID (onenmq conn ID) to session_ID/other info - do not persist: - room specific access: check every time it makes a request because it can change - values that admin level can change - - - """ diff --git a/sogs/routes/omq_subrequest.py b/sogs/routes/omq_subrequest.py new file mode 100644 index 00000000..72c19d4e --- /dev/null +++ b/sogs/routes/omq_subrequest.py @@ -0,0 +1,64 @@ +from ..web import app +from ..omq import omq_global + +from flask import request, g +from io import BytesIO +from typing import Optional, Union +import traceback, json, urllib.parse, oxenmq + + +def make_omq_subreq( + subreq_id, + endpoint: str, + query: str, + pubkey, + msg_later: oxenmq.Message.send_later, + params: Optional[Union[dict, list]] = None, + client_reauth: bool = False, +): + """ + Makes an omq subrequest from the given parameters, returns the response object and a dict of + lower-case response headers keys to header values + + Parameters: + subreq_id - randomly generated ID for subrequest + endpoint - the omq endpoint + query - the request query + pubkey - pk of client making request + msg_later - &oxenmq::Message::DeferredSend reference to be stored in subreq_queue along with data + params - a json value to dump as the the query parameters + """ + + if params is not None: + body = json.dumps(params, separators=(',', ':')).encode() + else: + body = b'' + + body_input = BytesIO(body) + content_length = len(body) + + subreq_body = {subreq_id:{ + 'endpoint':endpoint, + 'query':query, + 'pubkey':pubkey, + 'msg_later':msg_later, + 'params':params + } + } + + try: + app.logger.debug(f"Injecting sub-request for omq.{endpoint} {query}") + g.client_reauth = client_reauth + + omq_global.subreq_queue.put(subreq_body) + + try: + import uwsgi + except ModuleNotFoundError: + return + + uwsgi.signal(123) + + except Exception: + app.logger.warning(f"Sub-request for omq.{endpoint} {query} failed: {traceback.format_exc()}") + raise diff --git a/sogs/routes/onion_omq_request.py b/sogs/routes/onion_omq_request.py new file mode 100644 index 00000000..e4dd6706 --- /dev/null +++ b/sogs/routes/onion_omq_request.py @@ -0,0 +1,113 @@ +from flask import request, abort, Blueprint +from nacl.utils import random +import oxenmq +import json + +from ..web import app +from .. import crypto, http, utils + +from .omq_subrequest import make_omq_subreq + + +def handle_v4onion_omqreq_plaintext(msg: oxenmq.Message): + """ + Handles a decrypted v4 onion request; this injects a subrequest to process it then returns the + result of that subrequest. In contrast to v3, it is more efficient (particularly for binary + input or output) and allows using endpoints that return headers or bodies with non-2xx response + codes. + + Message (request) components: + + "endpoint" - the omq endpoint + "query" - the request query + "pubkey" - pk of client making request + "params" - a json value to dump as the the query parameters + + Example: + full request: `@omq.some_endpoint("room.messages.since", {'Room:room', 'int:seqno'})` + endpoint: 'some_endpoint' + query: 'room.messages.since' + pubkey: jvi0grsj3029fjwhatever + params: {'Room:room', 'int:seqno'} + """ + + try: + body = msg.data()[0] + + if not (body.startswith(b'l') and body.endswith(b'e')): + raise RuntimeError("Invalid onion request body: expected bencoded list") + + args = json.loads(body) + + subreq_id = random(16) + endpoint = args['endpoint'] + query = args['query'] + pubkey = utils.decode_hex_or_b64(args['pubkey'], 33) + params = args['params'] + + response = make_omq_subreq( + subreq_id, + endpoint, + query, + pubkey, + msg.later(), + params, + client_reauth=True, # Because onion requests have auth headers on the *inside* + ) + + data = response.get_data() + app.logger.debug( + f"Onion sub-request for {endpoint} returned {response.status_code}, {len(data)} bytes" + ) + + args = {'code': response.status_code, 'headers': headers} + + except Exception as e: + app.logger.warning("Invalid v4 onion request: {}".format(e)) + args = {'code': http.BAD_REQUEST, 'headers': {'content-type': 'text/plain; charset=utf-8'}} + data = b'Invalid v4 onion request' + + args = json.dumps(args).encode() + return b''.join( + (b'l', str(len(args)).encode(), b':', args, str(len(data)).encode(), b':', data, b'e') + ) + + +def decrypt_onionreq(): + try: + return crypto.parse_junk(request.data) + except Exception as e: + app.logger.warning("Failed to decrypt onion request: {}".format(e)) + abort(http.BAD_REQUEST) + + +def handle_v4_onion_request(): + """ + Parse a v4 onion request. See handle_v4_onionreq_plaintext(). + """ + + # Some less-than-ideal decisions in the onion request protocol design means that we are stuck + # dealing with parsing the request body here in the internal format that is meant for storage + # server, but the *last* hop's decrypted, encoded data has to get shared by us (and is passed on + # to us in its raw, encoded form). It looks like this: + # + # [N][blob][json] + # + # where N is the size of blob (4 bytes, little endian), and json contains *both* the elements + # that were meant for the last hop (like our host/port/protocol) *and* the elements that *we* + # need to decrypt blob (specifically: "ephemeral_key" and, optionally, "enc_type" [which can be + # used to use xchacha20-poly1305 encryption instead of AES-GCM]). + # + # The parse_junk here takes care of decoding and decrypting this according to the fields *meant + # for us* in the json (which include things like the encryption type and ephemeral key): + try: + junk = crypto.parse_junk(request.data) + except RuntimeError as e: + app.logger.warning("Failed to decrypt onion request: {}".format(e)) + abort(http.BAD_REQUEST) + + # On the way back out we re-encrypt via the junk parser (which uses the ephemeral key and + # enc_type that were specified in the outer request). We then return that encrypted binary + # payload as-is back to the client which bounces its way through the SN path back to the client. + response = handle_v4onion_omqreq_plaintext(junk.payload) + return junk.transformReply(response) diff --git a/sogs/routes/rooms.py b/sogs/routes/rooms.py index de0348cf..64b20732 100644 --- a/sogs/routes/rooms.py +++ b/sogs/routes/rooms.py @@ -1,5 +1,6 @@ from .. import config, db, http from ..model import room as mroom, exc, user as muser +from ..omq import omq_global, blueprints_global from ..web import app from . import auth @@ -13,6 +14,7 @@ rooms = Blueprint('rooms', __name__) +blueprints_global['rooms'] = rooms def get_room_info(room): diff --git a/sogs/routes/subrequest.py b/sogs/routes/subrequest.py index d856a5fd..c4cb5708 100644 --- a/sogs/routes/subrequest.py +++ b/sogs/routes/subrequest.py @@ -29,7 +29,7 @@ def make_subrequest( body - the bytes content of the body of a POST/PUT method. If specified then content_type will default to 'application/octet-stream'. json - a json value to dump as the body of the request. If specified then content_type will - default to 'applicaton/json'. + default to 'application/json'. user_reauth - if True then we allow user re-authentication on the subrequest based on its X-SOGS-* headers; if False (the default) then the user auth on the outer request is preserved (even if it was None) and inner request auth headers will be ignored. diff --git a/sogs/routes/users.py b/sogs/routes/users.py index 525ea158..73427088 100644 --- a/sogs/routes/users.py +++ b/sogs/routes/users.py @@ -3,6 +3,7 @@ from ..model.user import User from ..web import app from . import auth +from ..omq import omq_global, blueprints_global from flask import abort, jsonify, g, Blueprint, request @@ -10,6 +11,7 @@ users = Blueprint('users', __name__) +blueprints_global['users'] = users def extract_rooms_or_global(req, admin=True): diff --git a/sogs/routes/views.py b/sogs/routes/views.py index 8ee8adcf..ee5788c0 100644 --- a/sogs/routes/views.py +++ b/sogs/routes/views.py @@ -3,6 +3,7 @@ from .. import config, crypto, http from ..model.room import get_accessible_rooms from . import auth, converters # noqa: F401 +from ..omq import omq_global, blueprints_global from io import BytesIO @@ -18,6 +19,7 @@ views = Blueprint('views', __name__) +blueprints_global['views'] = views @views.get("/") diff --git a/sogs/utils.py b/sogs/utils.py index 9dbaf38e..0a54325e 100644 --- a/sogs/utils.py +++ b/sogs/utils.py @@ -4,7 +4,6 @@ import base64 - from flask import request, abort, Response import json from typing import Union, Tuple From 7dd7b5b3947d0693e9700205bc15ac005fe75837 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 20 Mar 2023 08:06:22 -0700 Subject: [PATCH 09/11] linting, CI fixes, circular import fix circular import issue --- sogs/model/clientmanager.py | 15 ++------- sogs/model/room.py | 1 - sogs/omq.py | 20 ++++------- sogs/routes/clients.py | 33 ++++++------------ sogs/routes/legacy.py | 9 +++-- sogs/routes/messages.py | 2 ++ sogs/routes/omq_auth.py | 57 ++++++++++++++++++-------------- sogs/routes/omq_subrequest.py | 21 +++++++----- sogs/routes/onion_omq_request.py | 2 +- sogs/utils.py | 4 +-- 10 files changed, 77 insertions(+), 87 deletions(-) diff --git a/sogs/model/clientmanager.py b/sogs/model/clientmanager.py index 5ec60016..49f7a048 100644 --- a/sogs/model/clientmanager.py +++ b/sogs/model/clientmanager.py @@ -79,26 +79,20 @@ class ClientManager: queue - BotQueue object """ - def __init__( - self, - *, - id: Optional[int] = None, - ) -> None: + def __init__(self, *, id: Optional[int] = None) -> None: self.id = id self.filter = False self.bqueue = BotQueue() self.clients = [] - def bqempty(self): return not self.bqueue._empty() - def register_client(self, conn_id, cid, authlevel, bot, priority): if not bot: # add client to self.clients return - + if not priority: # if no priority is given, lowest priority is assigned priority = self.qsize() @@ -108,19 +102,16 @@ def register_client(self, conn_id, cid, authlevel, bot, priority): priority += 1 self.bqueue._put(PriorityTuple(priority, bot)) - def deregister_client(self, cid, bot): if not bot: # remove client from clients list return - + # remove bot from bot queue - def peek(self, priority: int): return self.bqueue._peek(priority) - def check_permission_for( self, room: Room, diff --git a/sogs/model/room.py b/sogs/model/room.py index 3000a943..3132584d 100644 --- a/sogs/model/room.py +++ b/sogs/model/room.py @@ -154,7 +154,6 @@ def _refresh(self, *, id=None, token=None, row=None, perms=False): if perms or not hasattr(self, '_perm_cache'): self._perm_cache = {} - def __str__(self): """Returns `Room[token]` when converted to a str""" return f"Room[{self.token}]" diff --git a/sogs/omq.py b/sogs/omq.py index df5a86f4..85ce4dad 100644 --- a/sogs/omq.py +++ b/sogs/omq.py @@ -4,8 +4,8 @@ import oxenmq, queue from oxenc import bt_serialize, bt_deserialize -from mule import log_exceptions -from routes import omq_auth +from .mule import log_exceptions +from .routes import omq_auth from . import crypto, config from .postfork import postfork from .model.clientmanager import ClientManager @@ -36,7 +36,7 @@ def __init__(self): if uwsgi.mule_id() != 0: uwsgi.opt['mule'].setup_omq(self) return - + uwsgi.register_signal(123, 'internal', self.handle_proxied_omq_req) from .web import app # Imported here to avoid circular import @@ -52,18 +52,13 @@ def __init__(self): global omq_global omq_global = self - @log_exceptions def subreq_response(self): pass - @log_exceptions def handle_proxied_omq_req(self): - id, subreq_body = self.send_mule( - command='get_next_request', - prefix='internal' - ) + id, subreq_body = self.send_mule(command='get_next_request', prefix='internal') ''' @@ -71,7 +66,7 @@ def handle_proxied_omq_req(self): ''' - return + return @log_exceptions def get_next_request(self): @@ -79,7 +74,6 @@ def get_next_request(self): id = list(subreq_body.keys())[0] return id, subreq_body[id] - @log_exceptions def register_client(self, msg: oxenmq.Message): cid, authlevel, bot, priority = bt_deserialize(msg.data()[0]) @@ -87,14 +81,12 @@ def register_client(self, msg: oxenmq.Message): self.client_map[conn_id] = cid self.manager.register_client(msg) - @log_exceptions def deregister_client(self, msg: oxenmq.Message): cid, bot = bt_deserialize(msg.data()[0]) self.client_map.pop(cid) self.manager.deregister_client(cid, bot) - def send_mule(self, command, *args, prefix="worker."): """ Sends a command to the mule from a worker (or possibly from the mule itself). The command will @@ -102,7 +94,7 @@ def send_mule(self, command, *args, prefix="worker."): Any args will be bt-serialized and send as message parts. """ - + if prefix: command = prefix + command diff --git a/sogs/routes/clients.py b/sogs/routes/clients.py index daf4bd23..eca38e72 100644 --- a/sogs/routes/clients.py +++ b/sogs/routes/clients.py @@ -22,7 +22,7 @@ @omq_auth.first_request def register(cid): """ - Registers a client with SOGS OMQ instance. In this context, "client" refers to any entity + Registers a client with SOGS OMQ instance. In this context, "client" refers to any entity seeking to create an authenticated OMQ connection. This may be, but is not limited to, a user or a bot @@ -44,12 +44,11 @@ def register(cid): """ req = request.json - bot = utils.get_int_param('bot') # will set bot == 1 if key "bot" has value True + bot = utils.get_int_param('bot') # will set bot == 1 if key "bot" has value True authlevel = req.get('authlevel') priority = req.get('priority') - - client = (bot is 1)[register_client(cid, authlevel), - register_bot(cid, authlevel, priority)] + + client = (bot is 1)[register_client(cid, authlevel), register_bot(cid, authlevel, priority)] return client @@ -78,7 +77,7 @@ def register_bot(cid, authlevel, priority): authlevel=authlevel, bot=1, priority=priority, - prefix='handler' + prefix='handler', ) return client @@ -106,7 +105,7 @@ def register_client(cid, authlevel): authlevel=authlevel, bot=0, priority=None, - prefix='handler' + prefix='handler', ) return client @@ -125,7 +124,7 @@ def unregister(cid): - 'bot' (bool) : is bot or not """ - + bot = utils.get_int_param('bot') client = (bot)[unregister_client(cid), unregister_bot(cid)] @@ -137,25 +136,15 @@ def unregister(cid): @clients.delete("/client/registered/client/") def unregister_client(cid): - client = omq_global.send_mule( - command='unregister_client', - cid=cid, - bot=0, - prefix='handler' - ) - + client = omq_global.send_mule(command='unregister_client', cid=cid, bot=0, prefix='handler') + return client @clients.post("/bot/deregistered/bot/") @clients.delete("/bot/registered/bot/") def unregister_bot(cid): - - client = omq_global.send_mule( - command='unregister_bot', - cid=cid, - bot=1, - prefix='handler' - ) + + client = omq_global.send_mule(command='unregister_bot', cid=cid, bot=1, prefix='handler') return client diff --git a/sogs/routes/legacy.py b/sogs/routes/legacy.py index a03254fc..62c0e5ff 100644 --- a/sogs/routes/legacy.py +++ b/sogs/routes/legacy.py @@ -2,7 +2,6 @@ from werkzeug.exceptions import HTTPException from ..web import app from .. import crypto, config, db, http, utils -from ..omq import send_mule from ..utils import jsonify_with_base64 from ..model.room import Room, get_accessible_rooms, get_deletions_deprecated from ..model.user import User @@ -321,7 +320,13 @@ def handle_legacy_delete_messages(ids=None): ids = room.delete_posts(ids, user) if ids: - send_mule("messages_deleted", ids) + # avoid circular imports + try: + from ..omq import omq_global + except ModuleNotFoundError: + return + + omq_global.send_mule("messages_deleted", ids) return jsonify({'status_code': http.OK}) diff --git a/sogs/routes/messages.py b/sogs/routes/messages.py index 37bb41e0..d062f820 100644 --- a/sogs/routes/messages.py +++ b/sogs/routes/messages.py @@ -10,6 +10,7 @@ messages = Blueprint('messages', __name__) blueprints_global['messages'] = messages + def qs_reactors(): return utils.get_int_param('reactors', 4, min=0, max=20, truncate=True) @@ -314,6 +315,7 @@ def message_single(room: Room, msg_id): - add some decorator to this s.t. it routes it to the correct OMQ endpoint """ + @messages.post("/room//message") @auth.user_required def post_message(room: Room): diff --git a/sogs/routes/omq_auth.py b/sogs/routes/omq_auth.py index 36b446de..fac158e1 100644 --- a/sogs/routes/omq_auth.py +++ b/sogs/routes/omq_auth.py @@ -20,14 +20,15 @@ omq = Blueprint('endpoint', __name__) + def endpoint(f): - """ + """ Default endpoint for omq routes to pass requests to; constructs flask HTTP request Message (request) components: "blueprint" - the flask blueprint - "query" - the request query + "query" - the request query "pubkey" - pk of client making request "params" - a json value to dump as the the query parameters @@ -52,64 +53,68 @@ def abort_request(code, msg, warn=True): def require_client(): - """ - Requires that an authenticated client was found in the OMQ instance; aborts with - UNAUTHORIZED if the request has no client + """ + Requires that an authenticated client was found in the OMQ instance; aborts with + UNAUTHORIZED if the request has no client """ if g.client_id is None: abort_request(http.UNAUTHORIZED, 'OMQ client authentication required') def client_required(f): - """ - Decorator for an endpoint that requires a client; this calls require_client() at the - beginning of the request to abort the request as UNAUTHORIZED if the client has not been - previously authenticated + """ + Decorator for an endpoint that requires a client; this calls require_client() at the + beginning of the request to abort the request as UNAUTHORIZED if the client has not been + previously authenticated """ @wraps(f) def required_client_wrapper(*args, **kwargs): require_client() return f(*args, **kwargs) - + return required_client_wrapper def require_authlevel(admin=True): require_client() - if g.client_authlevel is not oxenmq.Authlevel.admin if admin else g.client_authlevel is not oxenmq.Authlevel.basic: + if ( + g.client_authlevel is not oxenmq.Authlevel.admin + if admin + else g.client_authlevel is not oxenmq.Authlevel.basic + ): abort_request( - http.FORBIDDEN, - f"This endpoint requires oxenmq.Authlevel.{'admin' if admin else 'basic'} permissions" + http.FORBIDDEN, + f"This endpoint requires oxenmq.Authlevel.{'admin' if admin else 'basic'} permissions", ) def basic_required(f): - """ Decorator for an endpoint that requires a client has basic OMQ authorization """ + """Decorator for an endpoint that requires a client has basic OMQ authorization""" @wraps(f) def required_basic_wrapper(*args, **kwargs): require_authlevel(admin=False) return f(*args, **kwargs) - + return required_basic_wrapper def admin_required(f): - """ Decorator for an endpoint that requires a client has admin OMQ authorization """ + """Decorator for an endpoint that requires a client has admin OMQ authorization""" @wraps(f) def required_admin_wrapper(*args, **kwargs): require_authlevel(admin=True) return f(*args, **kwargs) - + return required_admin_wrapper def first_request(f): - """ Decorator for an endpoint that will be the very first request for a given client. This + """Decorator for an endpoint that will be the very first request for a given client. This will ensure that the client is then registered for any subsequent requests. - + This function will typically take the folling parameters: - cid : unique client ID to be attributed - authlevel (oxenmq) @@ -119,7 +124,7 @@ def first_request(f): def first_request_wrapper(*args, cid, authlevel, **kwargs): handle_omq_registration(cid, authlevel) return f(*args, cid=cid, authlevel=authlevel, **kwargs) - + return first_request_wrapper @@ -128,9 +133,11 @@ def handle_omq_registration(sid, authlevel): Registers client with OMQ instance before its very first request """ if hasattr(g, 'client_id') and hasattr(g, 'client_authlevel') and not g.client_reauth: - app.logger.warning(f"Client {g.client_id} already registered for {g.client_authlevel} access") + app.logger.warning( + f"Client {g.client_id} already registered for {g.client_authlevel} access" + ) return - + """ Here goes ye olde OMQ registration logic. We need to decide what identification will be used to verify every connected client s.t. that information persists for all subsequent @@ -147,11 +154,13 @@ def verify_omq_auth(): """ Verifies OMQ authentication before each request """ - + # If there is already a g.o_id, then this is NOT the first request made by this client, unless # g.client_reauth has been specifically set if hasattr(g, 'client_id') and hasattr(g, 'client_authlevel') and not g.client_reauth: - app.logger.debug(f"Client {g.client_id} already authenticated for {g.client_authlevel} access") + app.logger.debug( + f"Client {g.client_id} already authenticated for {g.client_authlevel} access" + ) return diff --git a/sogs/routes/omq_subrequest.py b/sogs/routes/omq_subrequest.py index 72c19d4e..a008997b 100644 --- a/sogs/routes/omq_subrequest.py +++ b/sogs/routes/omq_subrequest.py @@ -23,7 +23,7 @@ def make_omq_subreq( Parameters: subreq_id - randomly generated ID for subrequest endpoint - the omq endpoint - query - the request query + query - the request query pubkey - pk of client making request msg_later - &oxenmq::Message::DeferredSend reference to be stored in subreq_queue along with data params - a json value to dump as the the query parameters @@ -37,12 +37,13 @@ def make_omq_subreq( body_input = BytesIO(body) content_length = len(body) - subreq_body = {subreq_id:{ - 'endpoint':endpoint, - 'query':query, - 'pubkey':pubkey, - 'msg_later':msg_later, - 'params':params + subreq_body = { + subreq_id: { + 'endpoint': endpoint, + 'query': query, + 'pubkey': pubkey, + 'msg_later': msg_later, + 'params': params, } } @@ -56,9 +57,11 @@ def make_omq_subreq( import uwsgi except ModuleNotFoundError: return - + uwsgi.signal(123) except Exception: - app.logger.warning(f"Sub-request for omq.{endpoint} {query} failed: {traceback.format_exc()}") + app.logger.warning( + f"Sub-request for omq.{endpoint} {query} failed: {traceback.format_exc()}" + ) raise diff --git a/sogs/routes/onion_omq_request.py b/sogs/routes/onion_omq_request.py index e4dd6706..65c24464 100644 --- a/sogs/routes/onion_omq_request.py +++ b/sogs/routes/onion_omq_request.py @@ -19,7 +19,7 @@ def handle_v4onion_omqreq_plaintext(msg: oxenmq.Message): Message (request) components: "endpoint" - the omq endpoint - "query" - the request query + "query" - the request query "pubkey" - pk of client making request "params" - a json value to dump as the the query parameters diff --git a/sogs/utils.py b/sogs/utils.py index 0a54325e..a83a92be 100644 --- a/sogs/utils.py +++ b/sogs/utils.py @@ -135,9 +135,9 @@ def get_int_param(name, default=None, *, required=False, min=None, max=None, tru return default try: - if (val is "true" or val is "True" or val is True): + if val == "true" or val == "True" or val == True: val = 1 - if (val is "false" or val is "False" or val is False): + if val == "false" or val == "False" or val == False: val = 0 val = int(val) except Exception: From 24fa200c79076eabeb6551a47c17b8b17f802d5b Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 20 Mar 2023 11:08:57 -0700 Subject: [PATCH 10/11] omq_auth.endpoint implemented to route omq subrequests to corresponding flask routes --- sogs/mule.py | 14 ----------- sogs/omq.py | 30 +++++++++++------------ sogs/routes/clients.py | 4 +-- sogs/routes/legacy.py | 2 +- sogs/routes/messages.py | 11 ++++----- sogs/routes/omq_auth.py | 42 +++++++++++++++----------------- sogs/routes/omq_subrequest.py | 4 +-- sogs/routes/onion_omq_request.py | 4 +-- sogs/routes/rooms.py | 5 ++-- sogs/routes/users.py | 4 +-- sogs/routes/views.py | 5 ++-- 11 files changed, 53 insertions(+), 72 deletions(-) diff --git a/sogs/mule.py b/sogs/mule.py index 265c3b9e..654243a2 100644 --- a/sogs/mule.py +++ b/sogs/mule.py @@ -109,17 +109,3 @@ def setup_omq(self, omq: OMQ): # it will work. (And so be careful not to recurse!) app.logger.debug("Mule connecting to self") omq.mule_conn = omq.connect_inproc(on_success=None, on_failure=self.inproc_fail) - - - -""" -TOFIX: oxenmq calls should pass as oxenmq.Message not as JSON - -def _on_subscribe(self, msg: oxenmq.Message): - code, message = None, None # If still None at the end, we send a reply - try: - args = json.loads(msg.data()[0]) - - pubkey = extract_hex_or_b64(args, "pubkey", 33) - -""" diff --git a/sogs/omq.py b/sogs/omq.py index 85ce4dad..7a83e74e 100644 --- a/sogs/omq.py +++ b/sogs/omq.py @@ -12,8 +12,6 @@ omq_global = None -global blueprints_global -blueprints_global = {} class OMQ: @@ -53,33 +51,35 @@ def __init__(self): omq_global = self @log_exceptions - def subreq_response(self): - pass + def subreq_response(self, msg: oxenmq.Message): + req_id, code, headers, data = bt_deserialize(msg.data()[0]) @log_exceptions def handle_proxied_omq_req(self): - id, subreq_body = self.send_mule(command='get_next_request', prefix='internal') + req_id, subreq_body = self.send_mule(command='get_next_request', prefix='internal') - ''' - - Handle omq subrequest - - ''' + # pass subrequest to omq endpoint + response, code = omq_auth.endpoint( + subreq_body['query'], subreq_body['pubkey'], subreq_body['params'] + ) - return + self.send_mule('subreq_response', req_id, code, response.headers, response.data) @log_exceptions def get_next_request(self): - subreq_body = self.subreq_queue.get() - id = list(subreq_body.keys())[0] - return id, subreq_body[id] + try: + subreq_body = self.subreq_queue.get() + except: + raise RuntimeError('No subrequest found in queue') + req_id = list(subreq_body.keys())[0] + return req_id, subreq_body[id] @log_exceptions def register_client(self, msg: oxenmq.Message): cid, authlevel, bot, priority = bt_deserialize(msg.data()[0]) conn_id = msg.conn() self.client_map[conn_id] = cid - self.manager.register_client(msg) + self.manager.register_client(conn_id, cid, authlevel, bot, priority) @log_exceptions def deregister_client(self, msg: oxenmq.Message): diff --git a/sogs/routes/clients.py b/sogs/routes/clients.py index eca38e72..12ca21d5 100644 --- a/sogs/routes/clients.py +++ b/sogs/routes/clients.py @@ -2,7 +2,7 @@ from ..model import room as mroom from ..model.user import User from ..web import app -from ..omq import omq_global, blueprints_global +from ..omq import omq_global from . import omq_auth from flask import abort, jsonify, g, Blueprint, request @@ -16,7 +16,7 @@ """ clients = Blueprint('clients', __name__) -blueprints_global['clients'] = clients +app.register_blueprint(clients) @omq_auth.first_request diff --git a/sogs/routes/legacy.py b/sogs/routes/legacy.py index 62c0e5ff..7eed1d30 100644 --- a/sogs/routes/legacy.py +++ b/sogs/routes/legacy.py @@ -325,7 +325,7 @@ def handle_legacy_delete_messages(ids=None): from ..omq import omq_global except ModuleNotFoundError: return - + omq_global.send_mule("messages_deleted", ids) return jsonify({'status_code': http.OK}) diff --git a/sogs/routes/messages.py b/sogs/routes/messages.py index d062f820..75f532e8 100644 --- a/sogs/routes/messages.py +++ b/sogs/routes/messages.py @@ -1,14 +1,15 @@ from .. import http, utils -from . import auth +from . import auth, omq_auth from model.room import Room -from ..omq import omq_global, blueprints_global +from ..omq import omq_global +from ..web import app from flask import abort, jsonify, g, Blueprint, request # Room message retrieving/submitting endpoints messages = Blueprint('messages', __name__) -blueprints_global['messages'] = messages +app.register_blueprint(messages) def qs_reactors(): @@ -367,8 +368,7 @@ def post_message(room: Room): """ req = request.json - msg = omq_global.send_mule( - command="send_to_handler", + msg = omq_global.manager.receive_message( user=g.user, room=room, data=utils.decode_base64(req.get('data')), @@ -376,7 +376,6 @@ def post_message(room: Room): whisper_to=req.get('whisper_to'), whisper_mods=bool(req.get('whisper_mods')), files=[int(x) for x in req.get('files', [])], - prefix="handler.", ) return utils.jsonify_with_base64(msg), http.CREATED diff --git a/sogs/routes/omq_auth.py b/sogs/routes/omq_auth.py index fac158e1..870dd47c 100644 --- a/sogs/routes/omq_auth.py +++ b/sogs/routes/omq_auth.py @@ -1,47 +1,43 @@ -import oxenmq -import auth +import oxenmq, importlib +from . import auth from ..web import app from ..db import query -from .. import config, crypto, http, utils -from ..omq import omq_global, blueprints_global -from ..model.user import User -from ..hashing import blake2b +from .. import http +from ..omq import omq_global +from typing import Callable from flask import request, abort, Response, g, Blueprint -import time -import nacl -from nacl.signing import VerifyKey -import nacl.exceptions -import nacl.bindings as sodium -import sqlalchemy.exc from functools import wraps # Authentication for handling OMQ requests -omq = Blueprint('endpoint', __name__) - -def endpoint(f): +def endpoint(query, pubkey, params): """ - Default endpoint for omq routes to pass requests to; constructs flask HTTP request + Default endpoint for omq requests to pass sub-requests to; passthrough for flask HTTP request Message (request) components: - "blueprint" - the flask blueprint "query" - the request query "pubkey" - pk of client making request "params" - a json value to dump as the the query parameters Example: - full request: `@omq.endpoint("messages", "room..messages.since.", {'Room:room', 'int:seqno'})` - blueprint: 'messages' - query: 'room..messages.since.' + full request: '@omq.endpoint('room.messages_since', {'Room:room', 'int:seqno'})' + query: 'room.messages_since' params: {'Room:room', 'int:seqno'} """ - @wraps(f) - def endpoint_wrapper(*args, blueprint, query, pubkey, params, **kwargs): - bp = blueprints_global['messages'] + assert ( + len(query.split('.')) == 2, + 'Error, query must be callable in format .', + ) + func = importlib.import_module(query) + + # unpack dictionary as request parameters + response, code = func(**params) + + return response, code def abort_request(code, msg, warn=True): diff --git a/sogs/routes/omq_subrequest.py b/sogs/routes/omq_subrequest.py index a008997b..10488d02 100644 --- a/sogs/routes/omq_subrequest.py +++ b/sogs/routes/omq_subrequest.py @@ -22,8 +22,8 @@ def make_omq_subreq( Parameters: subreq_id - randomly generated ID for subrequest - endpoint - the omq endpoint - query - the request query + endpoint - the flask blueprint/endpoint to be queried + query - the callable module method in format . pubkey - pk of client making request msg_later - &oxenmq::Message::DeferredSend reference to be stored in subreq_queue along with data params - a json value to dump as the the query parameters diff --git a/sogs/routes/onion_omq_request.py b/sogs/routes/onion_omq_request.py index 65c24464..e2a0230e 100644 --- a/sogs/routes/onion_omq_request.py +++ b/sogs/routes/onion_omq_request.py @@ -24,9 +24,9 @@ def handle_v4onion_omqreq_plaintext(msg: oxenmq.Message): "params" - a json value to dump as the the query parameters Example: - full request: `@omq.some_endpoint("room.messages.since", {'Room:room', 'int:seqno'})` + full request: `omq.endpoint('some_endpoint', 'room.messages_since', jvi0grsj3029fjwhatever, {'Room:room', 'int:seqno'})` endpoint: 'some_endpoint' - query: 'room.messages.since' + query: 'room.messages_since' pubkey: jvi0grsj3029fjwhatever params: {'Room:room', 'int:seqno'} """ diff --git a/sogs/routes/rooms.py b/sogs/routes/rooms.py index 64b20732..1cd2ceb0 100644 --- a/sogs/routes/rooms.py +++ b/sogs/routes/rooms.py @@ -1,6 +1,6 @@ from .. import config, db, http from ..model import room as mroom, exc, user as muser -from ..omq import omq_global, blueprints_global +from ..omq import omq_global from ..web import app from . import auth @@ -12,9 +12,8 @@ # Room-related routes, excluding retrieving/posting messages - rooms = Blueprint('rooms', __name__) -blueprints_global['rooms'] = rooms +app.register_blueprint(rooms) def get_room_info(room): diff --git a/sogs/routes/users.py b/sogs/routes/users.py index 73427088..9bc4c854 100644 --- a/sogs/routes/users.py +++ b/sogs/routes/users.py @@ -3,7 +3,7 @@ from ..model.user import User from ..web import app from . import auth -from ..omq import omq_global, blueprints_global +from ..omq import omq_global from flask import abort, jsonify, g, Blueprint, request @@ -11,7 +11,7 @@ users = Blueprint('users', __name__) -blueprints_global['users'] = users +app.register_blueprint(users) def extract_rooms_or_global(req, admin=True): diff --git a/sogs/routes/views.py b/sogs/routes/views.py index ee5788c0..b20b018d 100644 --- a/sogs/routes/views.py +++ b/sogs/routes/views.py @@ -3,7 +3,8 @@ from .. import config, crypto, http from ..model.room import get_accessible_rooms from . import auth, converters # noqa: F401 -from ..omq import omq_global, blueprints_global +from ..omq import omq_global +from ..web import app from io import BytesIO @@ -19,7 +20,7 @@ views = Blueprint('views', __name__) -blueprints_global['views'] = views +app.register_blueprint(views) @views.get("/") From 765ae95d44994c45650510a2a492fb05bd796f4f Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 20 Mar 2023 11:35:07 -0700 Subject: [PATCH 11/11] added comment block with next thing to figure out, misc --- sogs/omq.py | 2 +- sogs/routes/onion_omq_request.py | 20 +++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/sogs/omq.py b/sogs/omq.py index 7a83e74e..4460ff2f 100644 --- a/sogs/omq.py +++ b/sogs/omq.py @@ -35,7 +35,7 @@ def __init__(self): uwsgi.opt['mule'].setup_omq(self) return - uwsgi.register_signal(123, 'internal', self.handle_proxied_omq_req) + uwsgi.register_signal(123, 'worker', self.handle_proxied_omq_req) from .web import app # Imported here to avoid circular import diff --git a/sogs/routes/onion_omq_request.py b/sogs/routes/onion_omq_request.py index e2a0230e..dc070dff 100644 --- a/sogs/routes/onion_omq_request.py +++ b/sogs/routes/onion_omq_request.py @@ -45,7 +45,13 @@ def handle_v4onion_omqreq_plaintext(msg: oxenmq.Message): pubkey = utils.decode_hex_or_b64(args['pubkey'], 33) params = args['params'] - response = make_omq_subreq( + # TOFIX: omq subrequest is signaled for execution by uwsgi.signal, which does not + # return a value. In the standard onion request implementation, the response is + # returned with the headers. In the OMQ onion request implementation, this is + # handled in omq.handle_proxied_omq_req, which receives a response and code from + # omq_auth.endpoint, passing them through the mule to omq.subreq_response + + make_omq_subreq( subreq_id, endpoint, query, @@ -55,23 +61,11 @@ def handle_v4onion_omqreq_plaintext(msg: oxenmq.Message): client_reauth=True, # Because onion requests have auth headers on the *inside* ) - data = response.get_data() - app.logger.debug( - f"Onion sub-request for {endpoint} returned {response.status_code}, {len(data)} bytes" - ) - - args = {'code': response.status_code, 'headers': headers} - except Exception as e: app.logger.warning("Invalid v4 onion request: {}".format(e)) args = {'code': http.BAD_REQUEST, 'headers': {'content-type': 'text/plain; charset=utf-8'}} data = b'Invalid v4 onion request' - args = json.dumps(args).encode() - return b''.join( - (b'l', str(len(args)).encode(), b':', args, str(len(data)).encode(), b':', data, b'e') - ) - def decrypt_onionreq(): try: