Merge pull request #69 from majestrate/file-upload-2022-02-23

file upload endpoints

Author: jagerman

api.yaml

@@ -16,206 +16,6 @@ externalDocs:
   description: Find out more about the Oxen project
   url: http://oxen.io
 paths:
-  /room/{roomToken}/file:
-    post:
-      tags: [Files]
-      summary: "Uploads a file to a room."
-      description: >
-        Takes the request as binary in the body and takes other properties via submitted headers.
-        This saves space, particularly for large uploads.  The user must have upload and posting
-        permissions for the room.  The file will have a default lifetime of 1 hour, but that is
-        extended to 15 days when the containing message referencing the uploaded file is posted.
-        
-        
-        See also the `.../fileJson` endpoint for submitting via a json body.
-      parameters:
-      - $ref: "#/components/parameters/pathRoomToken"
-      - name: X-Filename
-        in: header
-        description: >
-          Suggested filename of the upload.  Typically the basename of the file uploaded from the
-          user.
-        schema:
-          type: string
-      requestBody:
-        description: The file content, in bytes.
-        required: true
-        content:
-          '*/*':
-            {}
-      responses:
-        200:
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  id:
-                    type: integer
-                    format: int64
-                    description: "The id of the file on the server."
-        403:
-          description: >
-            Upload forbidden.  This response code indicates that the user does not have posting
-            and/or upload permissions in the room either because of room settings, user restriction,
-            or because the user is banned.
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  banned:
-                    type: boolean
-                    description: >
-                      True if the upload was denied because the user is banned, omitted otherwise.
-                  noWrite:
-                    type: boolean
-                    description: >
-                      True if the upload was denied because the user does not have write access to
-                      the room (but is not banned).  Omitted otherwise.
-                  noUpload:
-                    type: boolean
-                    description: >
-                      True if the upload was denied because the user does not have upload access to
-                      the room (but is not banned and has write permissions).  Omitted otherwise.
-
-  /room/{roomToken}/fileJSON:
-    post:
-      tags: [Files]
-      summary: "Uploads a file to a room using a JSON encoded body."
-      description: >
-        This is less efficient when a binary upload is possible because the body must be passed as
-        base64-encoded data (which is 33% larger).  The user must have upload and posting
-        permissions for the room.  [NOT YET IMPLEMENTED: The file will have a default lifetime of 1
-          hour, but that is extended to 15 days when the containing message referencing the upload
-          is submitted.]
-      parameters:
-      - $ref: "#/components/parameters/pathRoomToken"
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              required: [filename, content]
-              properties:
-                filename:
-                  type: string
-                  description: "Suggested filename of the upload.  Typically the basename of the file uploaded from the user."
-                content:
-                  type: string
-                  format: byte
-                  description: The file content, in base64 encoding.
-      responses:
-        200:
-          $ref: "#/paths/~1room~1%7BroomToken%7D~1file/post/responses/200"
-        403:
-          $ref: "#/paths/~1room~1%7BroomToken%7D~1file/post/responses/403"
-
-  /room/{roomToken}/file/{fileId}:
-    get:
-      tags: [Files]
-      summary: "Retrieves a file from the room via JSON."
-      description: >
-        Retrieves a file via a fileId from the room via a JSON object response.  This is noticeably
-        less efficient (particularly for large files) than the binary version when making direct
-        requests because the file data must be encoded using base64 encoding.
-      parameters:
-        - $ref: "#/components/parameters/pathRoomToken"
-        - $ref: "#/components/parameters/pathFileId"
-      responses:
-        200:
-          description: successful operation; returns the file in JSON.
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  filename:
-                    type: string
-                    description: >
-                      The suggested filename of the file. Omitted if the file was uploaded without a
-                      filename (e.g. from older clients, or clients that specify an empty filename.)
-                  size:
-                    type: integer
-                    format: int64
-                    description: >
-                      The file size, in bytes. (*Not* the length of the base64-encoded data.)
-                  uploaded:
-                    type: number
-                    format: double
-                    description: The unix timestamp when the file was uploaded.
-                  expires:
-                    type: number
-                    format: double
-                    nullable: true
-                    description: >
-                      The unix timestamp when the file is scheduled to be removed.  Will be null if
-                      the attachment is permanent, such as for room images or attachments in pinned
-                      messages.
-        403:
-          $ref: "#/paths/~1room~1%7BroomToken%7D/get/responses/403"
-        404:
-          description: >
-            The referenced file does not exist.  (It may have expired, or may be invalid.)
-
-  /room/{roomToken}/file/{fileId}/{filename}:
-    get:
-      tags: [Files]
-      summary: "Retrieves a file from the room as binary."
-      description: >
-        Retrieves a file via a fileId from the room, returning the file content directly as the
-        binary response body.  The filename parameter is ignored and may be empty: it is primarily
-        included to aid in clients that want the request to include a filename, and differentiates
-        this as a request retrieving the file itself rather than the file as a JSON response.  See
-        the version without `/filename` for a JSON-returning version.
-      parameters:
-        - $ref: "#/components/parameters/pathRoomToken"
-        - $ref: "#/components/parameters/pathFileId"
-        - name: filename
-          in: path
-          required: true
-          description: >
-            Filename if known by the requesting client, and empty otherwise.  The value of this
-            parameter is ignored by the server itself: it is included to differentiate this request
-            from the JSON version, and so that clients may include a filename in the request URL for
-            contexts where that is useful.
-          schema:
-            type: integer
-            format: int64
-      responses:
-        200:
-          description: successful operation; returns the file, in raw bytes.
-          headers:
-            Content-Length:
-              description: The size of the file.
-              schema:
-                type: integer
-                format: int64
-              example: 12345
-            Date:
-              description: The HTTP timestamp at which the file was uploaded.
-              schema:
-                type: string
-                example: "Thu, 7 Oct 2021 00:42:00 GMT"
-            Expires:
-              description: >
-                The HTTP timestamp at which the file is scheduled to expire.  This header is omitted
-                if the attachment is non-expiring (e.g. for attachments in a pinned message [NOT YET
-                IMPLEMENTED]).
-              schema:
-                type: string
-                example: "Fri, 22 Oct 2021 00:42:42 GMT"
-          content:
-            application/octet-stream:
-              schema:
-                type: string
-                format: binary
-        403:
-          $ref: "#/paths/~1room~1%7BroomToken%7D~1file~1%7BfileId%7D/get/responses/403"
-        404:
-          $ref: "#/paths/~1room~1%7BroomToken%7D~1file~1%7BfileId%7D/get/responses/404"
   /user/{sessionId}/permission:
     post:
       tags: [Users]

conftest.py

@@ -12,6 +12,9 @@
 
 import sqlalchemy  # noqa: E402
 
+import atexit, shutil, tempfile  # noqa: E402 E401
+
+_tempdirs = set()
 
 sogs.omq.test_suite = True
 
@@ -91,7 +94,9 @@ def db(request, pgsql):
     value is the db module itself (so typically you don't import it at all but instead get it
     through this fixture, which also creates an empty db for you).
     """
-
+    d = tempfile.mkdtemp(prefix='tmp_pysogs')
+    _tempdirs.add(d)
+    config.UPLOAD_PATH = d
     trace = request.config.getoption("--sql-tracing")
 
     from sogs import db as db_
@@ -270,3 +275,5 @@ def no_rate_limit():
 
 
 web.app.config.update({'TESTING': True})
+
+atexit.register(lambda: [shutil.rmtree(d) for d in _tempdirs])

sogs/config.py

@@ -35,6 +35,7 @@
 REQUIRE_BLIND_KEYS = False
 TEMPLATE_PATH = 'templates'
 STATIC_PATH = 'static'
+UPLOAD_PATH = 'uploads'
 
 # Will be true if we're running as a uwsgi app, false otherwise; used where we need to do things
 # only in one case or another (e.g. database initialization only via app mode).
@@ -106,6 +107,7 @@ def bool_opt(name):
         'files': {
             'expiry': ('UPLOAD_DEFAULT_EXPIRY_DAYS', None, float),
             'max_size': ('UPLOAD_FILE_MAX_SIZE', None, int),
+            'uploads_dir': ('UPLOAD_PATH', path_exists, val_or_none),
         },
         'rooms': {
             'active_threshold': ('ROOM_DEFAULT_ACTIVE_THRESHOLD', None, float),

sogs/model/exc.py

@@ -18,7 +18,7 @@ def __init__(self, token):
 
 
 class NoSuchFile(NotFound):
-    """Thrown when trying to construct a File from a token that doesn't exist"""
+    """Thrown when trying to construct a File from an id that doesn't exist"""
 
     def __init__(self, id):
         self.id = id

sogs/model/file.py

@@ -13,12 +13,14 @@ class File:
         id - the numeric file id, i.e. primary key
         room - the Room that this file belongs to (only retrieved on demand).
         uploader - the User that uploaded this file (only retrieved on demand).
+        post_id - the id of the post to which this file is attached, None if unattached.
         size - the size (in bytes) of this file
         uploaded - unix timestamp when the file was uploaded
         expiry - unix timestamp when the file expires.  None for non-expiring files.
         path - the path of this file on disk, relative to the base data directory.
         filename - the suggested filename provided by the user.  None for there is no suggestion
-            (this will always be the case for files uploaded by legacy Session clients).
+            (this will always be the case for files uploaded by legacy Session clients, and
+            sometimes by newer Session clients, e.g. when uploading from a paste).
     """
 
     def __init__(self, row=None, *, id=None):
@@ -37,17 +39,28 @@ def __init__(self, row=None, *, id=None):
             self.id,
             self._fetch_room_id,
             self._fetch_uploader_id,
+            self.post_id,
             self.size,
             self.uploaded,
             self.expiry,
             self.filename,
             self.path,
         ) = (
             row[c]
-            for c in ('id', 'room', 'uploader', 'size', 'uploaded', 'expiry', 'filename', 'path')
+            for c in (
+                'id',
+                'room',
+                'uploader',
+                'message',
+                'size',
+                'uploaded',
+                'expiry',
+                'filename',
+                'path',
+            )
         )
-        self._uploader = None
         self._room = None
+        self._uploader = None
 
     @property
     def room(self):
@@ -72,7 +85,7 @@ def room_id(self):
         Accesses the id of the room to which this file was uploaded.  Equivalent to .room.id, except
         that we don't fetch/cache the Room row.
         """
-        return self._fetch_room_id if self._room is None else self._fetch_room_id
+        return self._fetch_room_id if self._room is None else self._room.id
 
     @property
     def uploader(self):

sogs/model/room.py

@@ -269,6 +269,9 @@ def image(self, file: Union[File, int]):
             if not isinstance(file, File):
                 file = File(id=file)
 
+            if file.room_id != self.id:
+                raise NoSuchFile(file.room_id)
+
             file.set_expiry(forever=True)
 
             if self.image:
@@ -1263,10 +1266,10 @@ def get_file(self, file_id: int):
                 r=self.id,
                 old_fid=file_id,
             ).first()
-
-        if row:
+        if not row:
+            return
+        if row['expiry'] is None or row['expiry'] > time.time():
             return File(row)
-        return None
 
     def upload_file(
         self,
@@ -1293,11 +1296,18 @@ def upload_file(
         if not self.check_upload(uploader):
             raise BadPermission()
 
-        files_dir = "uploads/" + self.token
+        files_dir = os.path.join(config.UPLOAD_PATH, self.token)
         os.makedirs(files_dir, exist_ok=True)
 
-        if filename is not None:
-            filename = re.sub(config.UPLOAD_FILENAME_BAD, "_", filename)
+        if filename is None:
+            upload_filename = None
+        else:
+            # nulls and / are prohibited characters on pretty much any system, so substitute them
+            # out for a REPLACEMENT CHARACTER (U+FFFD) if in the given filename.
+            filename = filename.replace('\0', '\uFFFD').replace('/', '\uFFFD')
+
+            # For the actual filename we write to disk we heavily sanitize:
+            upload_filename = re.sub(config.UPLOAD_FILENAME_BAD, "_", filename)
 
         file_id, file_path = None, None
 
@@ -1323,17 +1333,17 @@ def upload_file(
                     filename=filename,
                 )
 
-                if filename is None:
-                    filename = '(unnamed)'
+                if upload_filename is None:
+                    upload_filename = '(unnamed)'
 
-                if len(filename) > config.UPLOAD_FILENAME_MAX:
-                    filename = (
-                        filename[: config.UPLOAD_FILENAME_KEEP_PREFIX]
+                if len(upload_filename) > config.UPLOAD_FILENAME_MAX:
+                    upload_filename = (
+                        upload_filename[: config.UPLOAD_FILENAME_KEEP_PREFIX]
                         + "..."
-                        + filename[-config.UPLOAD_FILENAME_KEEP_SUFFIX :]
+                        + upload_filename[-config.UPLOAD_FILENAME_KEEP_SUFFIX :]
                     )
 
-                file_path = f"{files_dir}/{file_id}_{filename}"
+                file_path = f"{files_dir}/{file_id}_{upload_filename}"
 
                 with open(file_path, 'wb') as f:
                     f.write(content)