feat: add experimental support for additional file types (#1758)

Author: wp99cp

.prettierignore

@@ -15,6 +15,10 @@
 # ignore bag and mcap files
 **/*.mcap
 **/*.bag
+**/*.db3
+**/*.tum
+**/*.svo2
+**/*.yaml
 
 # ignore build info file
 frontend/src/build.ts

backend/src/services/file.service.ts

@@ -18,7 +18,7 @@ import {
     ConflictException,
     Injectable,
     NotFoundException,
-    OnModuleInit,
+    OnModuleInit, UnsupportedMediaTypeException,
 } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import jwt from 'jsonwebtoken';
@@ -519,9 +519,9 @@ export class FileService implements OnModuleInit {
 
         return await externalMinio.presignedUrl(
             'GET',
-            file.type === FileType.MCAP
-                ? env.MINIO_MCAP_BUCKET_NAME
-                : env.MINIO_BAG_BUCKET_NAME,
+            file.type === FileType.BAG
+                ? env.MINIO_BAG_BUCKET_NAME
+                : env.MINIO_MCAP_BUCKET_NAME,
             file.uuid, // we use the uuid as the filename in Minio
             expires ? 4 * 60 * 60 : 604_800, // 604800 seconds = 1 week
             {
@@ -792,15 +792,26 @@ export class FileService implements OnModuleInit {
 
                 logger.debug(`Creating temporary access for file: ${filename}`);
 
-                // verify that file has ending .bag or .mcap
-                if (!filename.endsWith('.bag') && !filename.endsWith('.mcap')) {
+                const fileExtensionToFileTypeMap: ReadonlyMap<string, FileType> = new Map([
+                    ['.bag', FileType.BAG],
+                    ['.mcap', FileType.MCAP],
+                    ['.yaml', FileType.YAML],
+                    ['.svo2', FileType.SVO2],
+                    ['.tum', FileType.TUM],
+                    ['.db3', FileType.DB3],
+                ]);
+
+                const supported_file_endings = [...fileExtensionToFileTypeMap.keys()];
+
+                if (!supported_file_endings.some((ending) => filename.endsWith(ending))) {
                     emptyCredentials.error = 'Invalid file ending';
                     return emptyCredentials;
                 }
 
-                const fileType: FileType = filename.endsWith('.bag')
-                    ? FileType.BAG
-                    : FileType.MCAP;
+                const matchingFileType = supported_file_endings.find(ending => filename.endsWith(ending));
+                if (matchingFileType === undefined) throw new UnsupportedMediaTypeException();
+                const fileType: FileType | undefined = fileExtensionToFileTypeMap.get(matchingFileType);
+                if (fileType === undefined) throw new UnsupportedMediaTypeException();
 
                 // check if file already exists
                 const existingFile = await this.fileRepository.exists({

backend/src/validation/validation-logic.ts

@@ -2,7 +2,7 @@ export const MISSION_NAME_REGEX = /^[\w\-_]{3,50}$/;
 
 export const PROJECT_NAME_REGEX = /^[\w\-_]{3,50}$/;
 
-export const FILE_NAME_REGEX = /^[\w\-.()]{3,50}.(bag|mcap)$/;
+export const FILE_NAME_REGEX = /^[\w\-.()]{3,50}.(bag|mcap|db3|yaml|svo2|tum)$/;
 
 export const NON_UUID_REGEX =
     /^(?![0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$)/;

cli/kleinkram/cli/_upload.py

@@ -44,6 +44,10 @@ def upload(
         False,
         help="fix filenames before upload, this does not change the filenames locally",
     ),
+    experimental_datatypes: bool = typer.Option(
+       False,
+       help="allow experimental datatypes (yaml, svo2, db3, tum)"
+    ),
     ignore_missing_tags: bool = typer.Option(False, help="ignore mission tags"),
 ) -> None:
     # get filepaths
@@ -61,6 +65,18 @@ def upload(
 
     if not fix_filenames:
         for file in file_paths:
+
+            # check for experimental datatypes and throw an exception if not allowed
+            EXPERIMENTAL_DATATYPES = {".yaml", ".svo2", ".db3", ".tum"}
+
+            if not experimental_datatypes:
+                if file.suffix.lower() in EXPERIMENTAL_DATATYPES:
+                    # NOTE: Assuming a 'DatatypeNotSupported' exception exists/is defined
+                    raise FileNameNotSupported(
+                        f"Datatype '{file.suffix}' for file {file} is not supported without the "
+                        f"`--experimental-datatypes` flag. "
+                    )
+
             if not kleinkram.utils.check_filename_is_sanatized(file.stem):
                 raise FileNameNotSupported(
                     f"Only `{''.join(kleinkram.utils.INTERNAL_ALLOWED_CHARS)}` are "

cli/kleinkram/utils.py

@@ -31,6 +31,10 @@
 SUPPORT_FILE_TYPES = [
     ".bag",
     ".mcap",
+    ".db3",
+    ".svo2",
+    ".tum",
+    ".yaml"
 ]
 
 

common/frontend_shared/enum.ts

@@ -100,6 +100,10 @@ export enum CookieNames {
 export enum FileType {
     BAG = 'BAG',
     MCAP = 'MCAP',
+    YAML = 'YAML',
+    SVO2 = 'SVO2',
+    TUM = 'TUM',
+    DB3 = 'DB3',
     ALL = 'ALL',
 }