Merge pull request #85 from jagerman/unlimited-file-expiry

Allow setting unlimited uploaded file expiry

Author: jagerman

sogs.ini.sample

@@ -59,7 +59,7 @@
 
 ; How long newly uploaded files should be stored before being cleaned up, in days.  Note that
 ; changing this only affects new files.  This limit does not apply to room images and attachments in
-; pinned messages, both of which do not expire.
+; pinned messages, both of which do not expire.  Can be set to 0 to never expire new uploads.
 ;
 ;expiry = 15
 

sogs/cleanup.py

@@ -68,7 +68,7 @@ def prune_files():
 def prune_message_history():
     count = query(
         "DELETE FROM message_history WHERE replaced < :t",
-        t=time.time() - config.MESSAGE_HISTORY_PRUNE_THRESHOLD * 86400,
+        t=time.time() - config.MESSAGE_HISTORY_PRUNE_THRESHOLD,
     ).rowcount
 
     if count > 0:
@@ -79,7 +79,7 @@ def prune_message_history():
 def prune_room_activity():
     count = query(
         "DELETE FROM room_users WHERE last_active < :t",
-        t=time.time() - config.ROOM_ACTIVE_PRUNE_THRESHOLD * 86400,
+        t=time.time() - config.ROOM_ACTIVE_PRUNE_THRESHOLD,
     ).rowcount
 
     if count > 0:

sogs/config.py

@@ -18,16 +18,16 @@
 OMQ_LISTEN = 'tcp://*:22028'
 OMQ_INTERNAL = 'ipc://./omq.sock'
 LOG_LEVEL = 'WARNING'
-DM_EXPIRY_DAYS = 15
-UPLOAD_DEFAULT_EXPIRY_DAYS = 15
+DM_EXPIRY = 15 * 86400.0  # Seconds, but specified in config file as days
+UPLOAD_DEFAULT_EXPIRY = 15 * 86400.0  # Seconds (or None), but specified in config file as days
 UPLOAD_FILENAME_MAX = 60
 UPLOAD_FILENAME_KEEP_PREFIX = 40
 UPLOAD_FILENAME_KEEP_SUFFIX = 17
 UPLOAD_FILE_MAX_SIZE = 6_000_000
 UPLOAD_FILENAME_BAD = re.compile(r"[^\w+\-.'()@\[\]]+")
-ROOM_ACTIVE_PRUNE_THRESHOLD = 60
-ROOM_DEFAULT_ACTIVE_THRESHOLD = 7
-MESSAGE_HISTORY_PRUNE_THRESHOLD = 30
+ROOM_ACTIVE_PRUNE_THRESHOLD = 60 * 86400.0  # Seconds, but specified in config file as days
+ROOM_DEFAULT_ACTIVE_THRESHOLD = 7 * 86400.0  # Seconds, but specified in config file as days
+MESSAGE_HISTORY_PRUNE_THRESHOLD = 30 * 86400.0  # Seconds, but specified in config file as days
 IMPORT_ADJUST_MS = 0
 PROFANITY_FILTER = False
 PROFANITY_SILENT = True
@@ -77,6 +77,12 @@ def path_exists(path):
     def val_or_none(v):
         return v or None
 
+    def days_to_seconds(v):
+        return float(v) * 86400.0
+
+    def days_to_seconds_or_none(v):
+        return days_to_seconds(v) if v else None
+
     truthy = ('y', 'yes', 'Y', 'Yes', 'true', 'True', 'on', 'On', '1')
     falsey = ('n', 'no', 'N', 'No', 'false', 'False', 'off', 'Off', '0')
     booly = truthy + falsey
@@ -105,18 +111,18 @@ def bool_opt(name):
             'http_show_recent': bool_opt('HTTP_SHOW_RECENT'),
         },
         'files': {
-            'expiry': ('UPLOAD_DEFAULT_EXPIRY_DAYS', None, float),
+            'expiry': ('UPLOAD_DEFAULT_EXPIRY', None, days_to_seconds_or_none),
             'max_size': ('UPLOAD_FILE_MAX_SIZE', None, int),
             'uploads_dir': ('UPLOAD_PATH', path_exists, val_or_none),
         },
         'rooms': {
-            'active_threshold': ('ROOM_DEFAULT_ACTIVE_THRESHOLD', None, float),
-            'active_prune_threshold': ('ROOM_ACTIVE_PRUNE_THRESHOLD', None, float),
+            'active_threshold': ('ROOM_DEFAULT_ACTIVE_THRESHOLD', None, days_to_seconds),
+            'active_prune_threshold': ('ROOM_ACTIVE_PRUNE_THRESHOLD', None, days_to_seconds),
         },
-        'direct_messages': {'expiry': ('DM_EXPIRY_DAYS', None, float)},
+        'direct_messages': {'expiry': ('DM_EXPIRY', None, days_to_seconds)},
         'users': {'require_blind_keys': bool_opt('REQUIRE_BLIND_KEYS')},
         'messages': {
-            'history_prune_threshold': ('MESSAGE_HISTORY_PRUNE_THRESHOLD', None, float),
+            'history_prune_threshold': ('MESSAGE_HISTORY_PRUNE_THRESHOLD', None, days_to_seconds),
             'profanity_filter': bool_opt('PROFANITY_FILTER'),
             'profanity_silent': bool_opt('PROFANITY_SILENT'),
             'profanity_custom': ('PROFANITY_CUSTOM', path_exists, val_or_none),

sogs/migrations/v_0_1_x.py

@@ -359,6 +359,10 @@ def ins_user(session_id):
                     )
                     timestamp = time.time()
 
+                exp = None
+                if config.UPLOAD_DEFAULT_EXPIRY:
+                    exp = timestamp + config.UPLOAD_DEFAULT_EXPIRY
+
                 new_id = db.insert_and_get_pk(
                     """
                     INSERT INTO files (room, size, uploaded, expiry, path)
@@ -368,7 +372,7 @@ def ins_user(session_id):
                     r=room_id,
                     size=size,
                     uploaded=timestamp,
-                    expiry=timestamp + 86400 * config.UPLOAD_DEFAULT_EXPIRY_DAYS,
+                    expiry=exp,
                     path=path,
                     dbconn=conn,
                 )
@@ -488,8 +492,8 @@ def ins_user(session_id):
             imported_activity, imported_active = 0, 0
 
             # Don't import rows we're going to immediately prune:
-            import_cutoff = time.time() - config.ROOM_ACTIVE_PRUNE_THRESHOLD * 86400
-            active_cutoff = time.time() - config.ROOM_DEFAULT_ACTIVE_THRESHOLD * 86400
+            import_cutoff = time.time() - config.ROOM_ACTIVE_PRUNE_THRESHOLD
+            active_cutoff = time.time() - config.ROOM_DEFAULT_ACTIVE_THRESHOLD
             n_activity = rconn.execute(
                 "SELECT COUNT(*) FROM user_activity WHERE last_active > ?", (import_cutoff,)
             ).fetchone()[0]

sogs/model/file.py

@@ -124,22 +124,25 @@ def read_base64(self):
     def set_expiry(self, duration=None, forever=False):
         """
         Updates the file expiry to `duration` seconds from now, or to unlimited if `forever` is
-        True.  If duration is None (and not using forever) then the default expiry will be used.
+        True.  If duration is None (and not using forever) then the default expiry (relative to the
+        current time) will be used.
         """
-        expiry = (
-            None
-            if forever
-            else time.time()
-            + (duration if duration is not None else config.UPLOAD_DEFAULT_EXPIRY_DAYS)
-        )
+        if forever:
+            expiry = None
+        elif duration is not None:
+            expiry = time.time() + duration
+        elif config.UPLOAD_DEFAULT_EXPIRY:
+            expiry = time.time() + config.UPLOAD_DEFAULT_EXPIRY
+        else:
+            expiry = None
         query("UPDATE files SET expiry = :when WHERE id = :f", when=expiry, f=self.id)
         self.expiry = expiry
 
     @staticmethod
     def reset_expiries(file_ids: List[int]):
         query(
             "UPDATE files SET expiry = uploaded + :exp WHERE id IN :ids",
-            exp=config.UPLOAD_DEFAULT_EXPIRY_DAYS * 86400.0,
+            exp=config.UPLOAD_DEFAULT_EXPIRY,
             ids=file_ids,
             bind_expanding=['ids'],
         )

sogs/model/message.py

@@ -36,7 +36,7 @@ def __init__(self, row=None, *, sender=None, recip=None, data=None):
                 sender=sender.id,
                 recipient=recip.id,
                 data=data,
-                expiry=time.time() + config.DM_EXPIRY_DAYS * 86400,
+                expiry=time.time() + config.DM_EXPIRY,
             )
         # sanity check
         assert row is not None

sogs/model/room.py

@@ -374,10 +374,10 @@ def default_upload(self, upload: bool):
                 query("UPDATE rooms SET upload = :upload WHERE id = :r", r=self.id, upload=upload)
                 self._refresh(perms=True)
 
-    def active_users(self, cutoff=config.ROOM_DEFAULT_ACTIVE_THRESHOLD * 86400):
+    def active_users(self, cutoff=config.ROOM_DEFAULT_ACTIVE_THRESHOLD):
         """
         Queries the number of active users in the past `cutoff` seconds.  Defaults to
-        config.ROOM_DEFAULT_ACTIVE_THRESHOLD days.  Note that room activity records are periodically
+        config.ROOM_DEFAULT_ACTIVE_THRESHOLD.  Note that room activity records are periodically
         removed, so going beyond config.ROOM_ACTIVE_PRUNE_THRESHOLD days is useless.
         """
 
@@ -656,6 +656,10 @@ def _own_files(self, msg_id: int, files: List[int], user):
         Associated any of the given file ids with the given message id.  Only files that are recent,
         expiring, unowned uploads by the same user are actually updated.
         """
+        expiry = None
+        if config.UPLOAD_DEFAULT_EXPIRY:
+            expiry = time.time() + config.UPLOAD_DEFAULT_EXPIRY
+
         return db.query(
             """
             UPDATE files SET
@@ -669,7 +673,7 @@ def _own_files(self, msg_id: int, files: List[int], user):
                 AND expiry IS NOT NULL
             """,
             m=msg_id,
-            exp=time.time() + config.UPLOAD_DEFAULT_EXPIRY_DAYS * 86400.0,
+            exp=expiry,
             ids=files,
             r=self.id,
             u=user.id,
@@ -1276,7 +1280,7 @@ def upload_file(
         uploader: User,
         *,
         filename: Optional[str] = None,
-        lifetime: Optional[float] = config.UPLOAD_DEFAULT_EXPIRY_DAYS * 86400.0,
+        lifetime: Optional[float] = config.UPLOAD_DEFAULT_EXPIRY,
     ):
         """
         Uploads a file to this room.  The uploader must have write and upload permissions.

sogs/routes/legacy.py

@@ -260,9 +260,7 @@ def handle_one_compact_poll(req):
     }
 
 
-def process_legacy_file_upload_for_room(
-    user, room, lifetime=config.UPLOAD_DEFAULT_EXPIRY_DAYS * 86400
-):
+def process_legacy_file_upload_for_room(user, room, lifetime=config.UPLOAD_DEFAULT_EXPIRY):
     """
     Uploads a file, posted by user, into the given room.  `lifetime` controls how long (in seconds)
     the file will be stored before expiry, and can be None for uploads (such as room images) that

sogs/routes/rooms.py

@@ -24,7 +24,7 @@ def get_room_info(room):
         'message_sequence': room.message_sequence,
         'created': room.created,
         'active_users': room.active_users(),
-        'active_users_cutoff': int(config.ROOM_DEFAULT_ACTIVE_THRESHOLD * 86400),
+        'active_users_cutoff': int(config.ROOM_DEFAULT_ACTIVE_THRESHOLD),
         'moderators': mods,
         'admins': admins,
         'read': room.check_read(g.user),

tests/test_dm.py

@@ -5,7 +5,7 @@
 from sogs.model.user import SystemUser
 import nacl.bindings as salt
 from nacl.utils import random
-import time
+from util import from_now
 
 
 def test_dm_default_empty(client, blind_user):
@@ -71,21 +71,21 @@ def test_dm_send(client, blind_user, blind_user2):
     r = sogs_post(client, f'/inbox/{blind_user2.session_id}', post, blind_user)
     assert r.status_code == 201
     data = r.json
-    assert -1 < data.pop('posted_at') - time.time() < 1
-    assert -1 < data.pop('expires_at') - config.DM_EXPIRY_DAYS * 86400 - time.time() < 1
+    assert data.pop('posted_at') == from_now.seconds(0)
+    assert data.pop('expires_at') == from_now.seconds(config.DM_EXPIRY)
     assert data == {k: v for k, v in msg_expected.items() if k != 'message'}
 
     r = sogs_get(client, '/inbox', blind_user2)
     assert r.status_code == 200
     assert len(r.json) == 1
     data = r.json[0]
-    assert -1 < data.pop('posted_at') - time.time() < 1
-    assert -1 < data.pop('expires_at') - config.DM_EXPIRY_DAYS * 86400 - time.time() < 1
+    assert data.pop('posted_at') == from_now.seconds(0)
+    assert data.pop('expires_at') == from_now.seconds(config.DM_EXPIRY)
     assert data == msg_expected
 
     r = sogs_get(client, '/outbox', blind_user)
     assert len(r.json) == 1
     data = r.json[0]
-    assert -1 < data.pop('posted_at') - time.time() < 1
-    assert -1 < data.pop('expires_at') - config.DM_EXPIRY_DAYS * 86400 - time.time() < 1
+    assert data.pop('posted_at') == from_now.seconds(0)
+    assert data.pop('expires_at') == from_now.seconds(config.DM_EXPIRY)
     assert data == msg_expected