Merge branch 'master' of github.com:scalableminds/webknossos into live-m3

Author: MichaelBuessemeyer

frontend/javascripts/viewer/view/right-border-tabs/comment_tab/comment_tab_view.tsx

@@ -9,7 +9,7 @@ import {
 } from "@ant-design/icons";
 import { Tree as AntdTree, Dropdown, type GetRef, Space, Tooltip, type TreeProps } from "antd";
 import type { EventDataNode } from "antd/es/tree";
-import { useLifecycle } from "beautiful-react-hooks";
+import useLifecycle from "beautiful-react-hooks/useLifecycle";
 import { InputKeyboard } from "libs/input";
 import { useEffectOnlyOnce } from "libs/react_hooks";
 import { useWkSelector } from "libs/react_hooks";

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala

@@ -21,6 +21,7 @@ import com.scalableminds.webknossos.datastore.models.UnfinishedUpload
 import com.scalableminds.webknossos.datastore.models.datasource.UsableDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON
 import com.scalableminds.webknossos.datastore.models.datasource._
 import com.scalableminds.webknossos.datastore.services.{DSRemoteWebknossosClient, DataSourceService}
+import com.scalableminds.webknossos.datastore.slacknotification.DSSlackNotificationService
 import com.scalableminds.webknossos.datastore.storage.{
   CredentialConfigReader,
   DataStoreRedisStore,
@@ -42,6 +43,7 @@ import java.net.URI
 import java.nio.file.{Files, Path}
 import scala.concurrent.{ExecutionContext, Future}
 import scala.jdk.FutureConverters._
+import scala.util.Try
 
 case class ReserveUploadInformation(
     uploadId: String, // upload id that was also used in chunk upload (this time without file paths)
@@ -105,6 +107,7 @@ class UploadService @Inject()(dataSourceService: DataSourceService,
                               remoteSourceDescriptorService: RemoteSourceDescriptorService,
                               exploreLocalLayerService: ExploreLocalLayerService,
                               dataStoreConfig: DataStoreConfig,
+                              slackNotificationService: DSSlackNotificationService,
                               val remoteWebknossosClient: DSRemoteWebknossosClient)(implicit ec: ExecutionContext)
     extends DatasetDeleter
     with DirectoryConstants
@@ -144,6 +147,8 @@ class UploadService @Inject()(dataSourceService: DataSourceService,
     s"upload___${uploadId}___datasetId"
   private def redisKeyForFilePaths(uploadId: String): String =
     s"upload___${uploadId}___filePaths"
+  private def redisKeyForReportedTooLargeUpload(uploadId: String): String =
+    s"upload___${uploadId}___tooLargeUpload"
 
   cleanUpOrphanUploads()
 
@@ -203,6 +208,7 @@ class UploadService @Inject()(dataSourceService: DataSourceService,
       )
       filePaths = Json.stringify(Json.toJson(reserveUploadInfo.filePaths.getOrElse(List.empty)))
       _ <- runningUploadMetadataStore.insert(redisKeyForFilePaths(reserveUploadInfo.uploadId), filePaths)
+      _ <- runningUploadMetadataStore.insert(redisKeyForReportedTooLargeUpload(reserveUploadInfo.uploadId), "false")
       _ <- runningUploadMetadataStore.insert(
         redisKeyForLinkedLayerIdentifier(reserveUploadInfo.uploadId),
         Json.stringify(Json.toJson(LinkedLayerIdentifiers(reserveUploadInfo.layersToLink)))
@@ -273,16 +279,23 @@ class UploadService @Inject()(dataSourceService: DataSourceService,
       (filePath, uploadDir) <- getFilePathAndDirOfUploadId(uploadFileId)
       isFileKnown <- runningUploadMetadataStore.contains(redisKeyForFileChunkCount(uploadId, filePath))
       totalFileSizeInBytesOpt <- runningUploadMetadataStore.findLong(redisKeyForTotalFileSizeInBytes(uploadId))
+      alreadyNotifiedAboutExceedingLimitOpt <- runningUploadMetadataStore.find(
+        redisKeyForReportedTooLargeUpload(uploadId))
       _ <- Fox.runOptional(totalFileSizeInBytesOpt) { maxFileSize =>
         runningUploadMetadataStore
           .increaseBy(redisKeyForCurrentUploadedTotalFileSizeInBytes(uploadId), currentChunkSize)
           .flatMap(newTotalFileSizeInBytesOpt => {
             if (newTotalFileSizeInBytesOpt.getOrElse(0L) > maxFileSize) {
+              runningUploadMetadataStore.insert(redisKeyForReportedTooLargeUpload(uploadId), "true")
               logger.warn(
                 s"Received upload chunk for $datasetId that pushes total file size to ${newTotalFileSizeInBytesOpt
-                  .getOrElse(0L)}, which is more than reserved $maxFileSize. Aborting the upload.")
-              cleanUpDatasetExceedingSize(uploadDir, uploadId).flatMap(_ =>
-                Fox.failure("dataset.upload.moreBytesThanReserved"))
+                  .getOrElse(0L)}, which is more than reserved $maxFileSize. Allowing upload for now.")
+              if (!alreadyNotifiedAboutExceedingLimitOpt.exists(s => Try(s.toBoolean).getOrElse(false))) {
+                slackNotificationService.noticeTooLargeUploadChunkRequest(
+                  s"Received upload chunk for $datasetId that pushes total file size to ${newTotalFileSizeInBytesOpt
+                    .getOrElse(0L)}, which is more than reserved $maxFileSize.")
+              }
+              Fox.successful(())
             } else {
               Fox.successful(())
             }
@@ -339,18 +352,6 @@ class UploadService @Inject()(dataSourceService: DataSourceService,
   private def uploadFullName(uploadId: String, datasetId: ObjectId, dataSourceId: DataSourceId) =
     s"upload $uploadId of dataset $datasetId ($dataSourceId)"
 
-  private def assertWithinRequestedFileSizeAndCleanUpOtherwise(uploadDir: Path, uploadId: String): Fox[Unit] =
-    for {
-      totalFileSizeInBytesOpt <- runningUploadMetadataStore.find(redisKeyForTotalFileSizeInBytes(uploadId))
-      _ <- Fox.runOptional(totalFileSizeInBytesOpt) { maxFileSize =>
-        tryo(FileUtils.sizeOfDirectoryAsBigInteger(uploadDir.toFile).longValue).toFox.map(actualFileSize =>
-          if (actualFileSize > maxFileSize.toLong) {
-            cleanUpDatasetExceedingSize(uploadDir, uploadId)
-            Fox.failure(s"Uploaded dataset exceeds the maximum allowed size of $maxFileSize bytes")
-          } else Fox.successful(()))
-      }
-    } yield ()
-
   def finishUpload(uploadInformation: UploadInformation, datasetId: ObjectId)(implicit tc: TokenContext): Fox[Unit] = {
     val uploadId = uploadInformation.uploadId
 
@@ -361,11 +362,25 @@ class UploadService @Inject()(dataSourceService: DataSourceService,
       needsConversion = uploadInformation.needsConversion.getOrElse(false)
       uploadDir = uploadDirectoryFor(dataSourceId.organizationId, uploadId)
       _ <- backupRawUploadedData(uploadDir, uploadBackupDirectoryFor(dataSourceId.organizationId, uploadId), datasetId).toFox
-      _ <- assertWithinRequestedFileSizeAndCleanUpOtherwise(uploadDir, uploadId)
+      // Temporarily disabled till reserved size for upload is fixed.
+      // _ <- assertWithinRequestedFileSizeAndCleanUpOtherwise(uploadDir, uploadId)
       _ <- checkAllChunksUploaded(uploadId)
       unpackToDir = unpackToDirFor(dataSourceId)
       _ <- ensureDirectoryBox(unpackToDir.getParent).toFox ?~> "dataset.import.fileAccessDenied"
       unpackResult <- unpackDataset(uploadDir, unpackToDir, datasetId).shiftBox
+      reservedTotalFileSizeInBytesOpt <- runningUploadMetadataStore.findLong(redisKeyForTotalFileSizeInBytes(uploadId))
+      actualUploadedFileSizeInBytesOpt <- runningUploadMetadataStore.findLong(
+        redisKeyForCurrentUploadedTotalFileSizeInBytes(uploadId))
+      // Logging successful uploads which exceeded size to notice how large the difference is. Should be removed later.
+      _ = reservedTotalFileSizeInBytesOpt.foreach(reservedBytes =>
+        actualUploadedFileSizeInBytesOpt.foreach(actualBytes => {
+          if (actualBytes > reservedBytes) {
+            logger.warn(
+              s"Finished upload for $datasetId that exceeded reserved upload size. $reservedBytes bytes were reserved but $actualBytes were uploaded according to redis store.")
+            slackNotificationService.noticeTooLargeUploadChunkRequest(
+              s"Finished upload for $datasetId that exceeded reserved upload size. $reservedBytes bytes were reserved but $actualBytes were uploaded according to redis store.")
+          }
+        }))
       _ <- cleanUpUploadedDataset(uploadDir, uploadId, reason = "Upload complete, data unpacked.")
       _ <- cleanUpOnFailure(unpackResult,
                             datasetId,
@@ -819,13 +834,6 @@ class UploadService @Inject()(dataSourceService: DataSourceService,
       }
     } yield ()
 
-  private def cleanUpDatasetExceedingSize(uploadDir: Path, uploadId: String): Fox[Unit] =
-    for {
-      datasetId <- getDatasetIdByUploadId(uploadId)
-      _ <- cleanUpUploadedDataset(uploadDir, uploadId, reason = "Exceeded reserved fileSize")
-      _ <- remoteWebknossosClient.deleteDataset(datasetId)
-    } yield ()
-
   private def removeFromRedis(uploadId: String): Fox[Unit] =
     for {
       _ <- runningUploadMetadataStore.remove(redisKeyForFileCount(uploadId))
@@ -845,6 +853,7 @@ class UploadService @Inject()(dataSourceService: DataSourceService,
       _ <- runningUploadMetadataStore.remove(redisKeyForLinkedLayerIdentifier(uploadId))
       _ <- runningUploadMetadataStore.remove(redisKeyForUploadId(dataSourceId))
       _ <- runningUploadMetadataStore.remove(redisKeyForFilePaths(uploadId))
+      _ <- runningUploadMetadataStore.remove(redisKeyForReportedTooLargeUpload(uploadId))
     } yield ()
 
   private def cleanUpOrphanUploads(): Fox[Unit] =

webknossos-datastore/app/com/scalableminds/webknossos/datastore/slacknotification/DSSlackNotificationService.scala

@@ -28,4 +28,10 @@ class DSSlackNotificationService @Inject()(rpc: RPC, config: DataStoreConfig) ex
       title = "Failed upload request",
       msg = msg
     )
+
+  def noticeTooLargeUploadChunkRequest(msg: String): Unit =
+    slackClient.warn(
+      title = "Too large total upload",
+      msg = msg
+    )
 }