Merge branch 'master' into s3-upload

frcroth · web-flow · commit a62c986bbc12 · 2025-09-11T18:19:16.000+02:00
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@ WEBKNOSSOS is an open-source tool for annotating and exploring large 3D image da
 * Scale data reconstruction projects with crowdsourcing workflows
 * Share datasets and annotations with collaborating scientists
 
-[Start using WEBKNOSSOS](https://webknossos.org) - [On your own server](https://docs.webknossos.org/webknossos/installation.html) - [User Documentation](https://docs.webknossos.org) - [Contact us](mailto:hello@webknossos.org)
+[Start using WEBKNOSSOS](https://webknossos.org) - [On your own server](https://docs.webknossos.org/webknossos/open_source/installation.html) - [User Documentation](https://docs.webknossos.org) - [Contact us](mailto:hello@webknossos.org)
 
 [![](https://img.shields.io/circleci/project/github/scalableminds/webknossos/master.svg?logo=circleci)](https://circleci.com/gh/scalableminds/webknossos)
 [![](https://img.shields.io/github/release/scalableminds/webknossos.svg)](https://github.com/scalableminds/webknossos/releases/latest)
diff --git a/app/WebknossosModule.scala b/app/WebknossosModule.scala
@@ -3,8 +3,8 @@ import com.scalableminds.webknossos.datastore.storage.DataVaultService
 import controllers.{Application, InitialDataService}
 import files.WkTempFileService
 import mail.MailchimpTicker
-import models.analytics.AnalyticsSessionService
-import models.annotation.{AnnotationMutexService, AnnotationStore, AnnotationDataSourceTemporaryStore}
+import models.analytics.{AnalyticsService, AnalyticsSessionService}
+import models.annotation.{AnnotationDataSourceTemporaryStore, AnnotationMutexService, AnnotationStore}
 import models.dataset.{DatasetService, ThumbnailCachingService}
 import models.job.{JobService, WorkerLivenessService}
 import models.organization.FreeCreditTransactionService
@@ -43,5 +43,6 @@ class WebknossosModule extends AbstractModule {
     bind(classOf[AnnotationDataSourceTemporaryStore]).asEagerSingleton()
     bind(classOf[CertificateValidationService]).asEagerSingleton()
     bind(classOf[FreeCreditTransactionService]).asEagerSingleton()
+    bind(classOf[AnalyticsService]).asEagerSingleton()
   }
 }
diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala
@@ -1,6 +1,5 @@
 package controllers
 
-import com.scalableminds.util.mvc.ApiVersioning
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.typesafe.config.ConfigRenderOptions
 import mail.{DefaultMails, Send}
@@ -12,43 +11,29 @@ import play.api.mvc.{Action, AnyContent, Result}
 import play.silhouette.api.Silhouette
 import security.{CertificateValidationService, WkEnv}
 import utils.sql.{SimpleSQLDAO, SqlClient}
-import utils.{StoreModules, WkConf}
+import utils.{BuildInfoService, WkConf}
 
 import javax.inject.Inject
 import scala.concurrent.ExecutionContext
 
 class Application @Inject()(actorSystem: ActorSystem,
                             userService: UserService,
-                            releaseInformationDAO: ReleaseInformationDAO,
+                            buildInfoService: BuildInfoService,
                             organizationDAO: OrganizationDAO,
                             conf: WkConf,
                             defaultMails: DefaultMails,
-                            storeModules: StoreModules,
                             sil: Silhouette[WkEnv],
                             certificateValidationService: CertificateValidationService)(implicit ec: ExecutionContext)
-    extends Controller
-    with ApiVersioning {
+    extends Controller {
 
   private lazy val Mailer =
     actorSystem.actorSelection("/user/mailActor")
 
   // Note: This route is used by external applications, keep stable
   def buildInfo: Action[AnyContent] = sil.UserAwareAction.async {
     for {
-      schemaVersion <- releaseInformationDAO.getSchemaVersion.futureBox
-    } yield {
-      addRemoteOriginHeaders(
-        Ok(
-          Json.obj(
-            "webknossos" -> Json.toJson(
-              webknossos.BuildInfo.toMap.view.mapValues(_.toString).filterKeys(_ != "certificatePublicKey").toMap),
-            "schemaVersion" -> schemaVersion.toOption,
-            "httpApiVersioning" -> apiVersioningInfo,
-            "localDataStoreEnabled" -> storeModules.localDataStoreEnabled,
-            "localTracingStoreEnabled" -> storeModules.localTracingStoreEnabled
-          ))
-      )
-    }
+      buildInfoJson <- buildInfoService.buildInfoJson
+    } yield addRemoteOriginHeaders(Ok(buildInfoJson))
   }
 
   // This only changes on server restart, so we can cache the full result.
diff --git a/app/controllers/JobController.scala b/app/controllers/JobController.scala
@@ -31,7 +31,7 @@ object MovieResolutionSetting extends ExtendedEnumeration {
 }
 
 object CameraPositionSetting extends ExtendedEnumeration {
-  val MOVING, STATIC_XZ, STATIC_YZ = Value
+  val MOVING, STATIC_ISOMETRIC, STATIC_XY, STATIC_XZ, STATIC_YZ = Value
 }
 
 case class AnimationJobOptions(
@@ -378,7 +378,6 @@ class JobController @Inject()(jobDAO: JobDAO,
           organization <- organizationDAO.findOne(dataset._organization)(GlobalAccessContext) ?~> Messages(
             "organization.notFound",
             dataset._organization)
-          _ <- Fox.fromBool(request.identity._organization == organization._id) ?~> "job.exportTiff.notAllowed.organization" ~> FORBIDDEN
           _ <- Fox.runOptional(layerName)(datasetService.assertValidLayerNameLax)
           _ <- Fox.runOptional(annotationLayerName)(datasetService.assertValidLayerNameLax)
           _ <- jobService.assertBoundingBoxLimits(bbox, mag)
@@ -496,7 +495,6 @@ class JobController @Inject()(jobDAO: JobDAO,
           organization <- organizationDAO.findOne(dataset._organization)(GlobalAccessContext) ?~> Messages(
             "organization.notFound",
             dataset._organization)
-          _ <- Fox.fromBool(request.identity._organization == organization._id) ?~> "job.renderAnimation.notAllowed.organization" ~> FORBIDDEN
           userOrganization <- organizationDAO.findOne(request.identity._organization)
           animationJobOptions = request.body
           _ <- Fox.runIf(!PricingPlan.isPaidPlan(userOrganization.pricingPlan)) {
diff --git a/app/models/analytics/AnalyticsEvent.scala b/app/models/analytics/AnalyticsEvent.scala
@@ -230,3 +230,15 @@ case class FrontendAnalyticsEvent(user: User, eventType: String, eventProperties
   override def eventProperties(analyticsLookUpService: AnalyticsLookUpService): Fox[JsObject] =
     Fox.successful(eventProperties ++ Json.obj("is_frontend_event" -> true))
 }
+
+case class WebknossosHeartbeatAnalyticsEvent(user: User, buildInfoJson: JsObject)(implicit ec: ExecutionContext)
+    extends AnalyticsEvent {
+  def eventType: String = "webknossos_heartbeat"
+  def eventProperties(analyticsLookUpService: AnalyticsLookUpService): Fox[JsObject] =
+    Fox.successful(
+      Json.obj(
+        "build_info" -> buildInfoJson,
+      )
+    )
+
+}
diff --git a/app/models/analytics/AnalyticsService.scala b/app/models/analytics/AnalyticsService.scala
@@ -8,9 +8,12 @@ import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.typesafe.scalalogging.LazyLogging
 import models.user.{MultiUserDAO, UserDAO}
 import com.scalableminds.util.tools.Box.tryo
+import com.scalableminds.webknossos.datastore.helpers.IntervalScheduler
+import org.apache.pekko.actor.ActorSystem
 import play.api.http.Status.UNAUTHORIZED
+import play.api.inject.ApplicationLifecycle
 import play.api.libs.json._
-import utils.WkConf
+import utils.{BuildInfoService, WkConf}
 
 import javax.inject.Inject
 import scala.concurrent.ExecutionContext
@@ -20,8 +23,13 @@ class AnalyticsService @Inject()(rpc: RPC,
                                  wkConf: WkConf,
                                  analyticsLookUpService: AnalyticsLookUpService,
                                  analyticsSessionService: AnalyticsSessionService,
-                                 analyticsDAO: AnalyticsDAO)(implicit ec: ExecutionContext)
+                                 analyticsDAO: AnalyticsDAO,
+                                 userDAO: UserDAO,
+                                 buildInfoService: BuildInfoService,
+                                 val actorSystem: ActorSystem,
+                                 val lifecycle: ApplicationLifecycle)(implicit val ec: ExecutionContext)
     extends LazyLogging
+    with IntervalScheduler
     with FoxImplicits {
 
   private lazy val conf = wkConf.BackendAnalytics
@@ -75,6 +83,17 @@ class AnalyticsService @Inject()(rpc: RPC,
     }
     Fox.successful(())
   }
+
+  override protected def tickerInterval: FiniteDuration = 24 hours
+
+  override protected def tick(): Fox[_] =
+    for {
+      oldestUser <- userDAO.findOldestActive
+      buildInfoJson <- Fox.fromFuture(buildInfoService.buildInfoJson)
+      event = WebknossosHeartbeatAnalyticsEvent(oldestUser, buildInfoJson)
+      _ = track(event)
+    } yield ()
+
 }
 
 class AnalyticsLookUpService @Inject()(userDAO: UserDAO, multiUserDAO: MultiUserDAO, wkConf: WkConf)
diff --git a/app/models/job/JobCommand.scala b/app/models/job/JobCommand.scala
@@ -12,8 +12,10 @@ object JobCommand extends ExtendedEnumeration {
    */
 
   val compute_mesh_file, compute_segment_index_file, convert_to_wkw, export_tiff, find_largest_segment_id,
-  globalize_floodfills, infer_nuclei, infer_neurons, infer_instances, materialize_volume_annotation, render_animation,
-  infer_mitochondria, align_sections, train_model, infer_with_model, train_neuron_model, train_instance_model = Value
+  materialize_volume_annotation, render_animation, align_sections, infer_nuclei, infer_neurons, infer_instances,
+  infer_mitochondria, train_neuron_model, train_instance_model,
+  // No-longer supported jobs, kept here to be able to display old existing jobs:
+  globalize_floodfills, train_model, infer_with_model = Value
 
   val highPriorityJobs: Set[Value] = Set(convert_to_wkw, export_tiff)
   val lowPriorityJobs: Set[Value] = values.diff(highPriorityJobs)
diff --git a/app/models/user/User.scala b/app/models/user/User.scala
@@ -168,6 +168,13 @@ class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
   def findIdsByMultiUserId(multiUserId: ObjectId): Fox[Seq[ObjectId]] =
     run(q"SELECT _id FROM $existingCollectionName WHERE _multiUser = $multiUserId".as[ObjectId])
 
+  def findOldestActive: Fox[User] =
+    for {
+      r <- run(
+        q"SELECT $columns FROM $existingCollectionName WHERE NOT isDeactivated ORDER BY created LIMIT 1".as[UsersRow])
+      parsed <- parseFirst(r, "oldestActive")
+    } yield parsed
+
   def buildSelectionPredicates(isEditableOpt: Option[Boolean],
                                isTeamManagerOrAdminOpt: Option[Boolean],
                                isAdminOpt: Option[Boolean],
diff --git a/app/utils/BuildInfoService.scala b/app/utils/BuildInfoService.scala
@@ -0,0 +1,25 @@
+package utils
+
+import com.scalableminds.util.mvc.ApiVersioning
+import controllers.ReleaseInformationDAO
+import play.api.libs.json.{JsObject, Json}
+
+import javax.inject.Inject
+import scala.concurrent.{ExecutionContext, Future}
+
+class BuildInfoService @Inject()(releaseInformationDAO: ReleaseInformationDAO, storeModules: StoreModules)
+    extends ApiVersioning {
+
+  def buildInfoJson(implicit ec: ExecutionContext): Future[JsObject] =
+    for {
+      schemaVersion <- releaseInformationDAO.getSchemaVersion.futureBox
+    } yield
+      Json.obj(
+        "webknossos" -> Json.toJson(
+          webknossos.BuildInfo.toMap.view.mapValues(_.toString).filterKeys(_ != "certificatePublicKey").toMap),
+        "schemaVersion" -> schemaVersion.toOption,
+        "httpApiVersioning" -> apiVersioningInfo,
+        "localDataStoreEnabled" -> storeModules.localDataStoreEnabled,
+        "localTracingStoreEnabled" -> storeModules.localTracingStoreEnabled
+      )
+}
diff --git a/conf/messages b/conf/messages
@@ -344,13 +344,11 @@ job.inferNeurons.notAllowed.organization = Currently neuron inferral is only all
 job.inferNeurons.annotationIdEvalParamsMissing=A evaluation of the neuron inferral jobs was requested but no annotation was supplied.
 job.inferMitochondria.notAllowed.onlySuperUsers=Currently mitochondria inferral is only allowed for super users.
 job.meshFile.notAllowed.organization = Calculating mesh files is only allowed for datasets of your own organization.
-job.exportTiff.notAllowed.organization=Exporting tiff files is only allowed for datasets of your own organization.
 job.segmentIndexFile.notAllowed.organization = Calculating segment index files is only allowed for datasets of your own organization.
 job.globalizeFloodfill.notAllowed.organization = Globalizing floodfills is only allowed for datasets of your own organization.
 job.materializeVolumeAnnotation.notAllowed.organization = Materializing volume annotations is only allowed for datasets of your own organization.
 job.noWorkerForDatastoreAndJob = No webknossos-worker supporting the requested job is available for the selected datastore.
 job.emailNotifactionsDisabled = Email notifications are not enabled for this job type.
-job.renderAnimation.notAllowed.organization = Rendering animations is only allowed for datasets of your own organization.
 job.alignSections.notAllowed.organization = Aligning sections is only allowed for datasets of your own organization.
 job.additionalCoordinates.invalid = The passed additional coordinates are invalid.
 job.trainModel.notAllowed.organization = Training AI models is only allowed for datasets of your own organization.
diff --git a/docs/automation/animations.md b/docs/automation/animations.md
@@ -12,7 +12,7 @@ Creating an animation is easy:
 2. Optionally, load some [3D meshes](../meshes/index.md) for any segments that you wish to highlight.
 3. For larger datasets, use the bounding box tool to create a bounding box around your area of interest. Smaller datasets can be used in their entirety.
 4. From the `Menu` dropdown in navbar at the top of the screen, select "Create Animation".
-5. Configure the animation options as desired, i.e. camera movement or resolution.
+5. Configure the animation options as desired, i.e. camera movement and positioning or video resolution.
 6. Click the `Start animation` button to launch the animation creation.
 
 
diff --git a/frontend/javascripts/admin/rest_api.ts b/frontend/javascripts/admin/rest_api.ts
@@ -1880,7 +1880,7 @@ window.setMaintenance = setMaintenance;
 // receives too many parameters, since this doesn't play well with the saga typings.
 type MeshRequest = {
   // The position is in voxels in mag 1
-  position: Vector3;
+  positionWithPadding: Vector3;
   additionalCoordinates: AdditionalCoordinate[] | undefined;
   mag: Vector3;
   segmentId: number; // Segment to build mesh for
@@ -1899,8 +1899,15 @@ export function computeAdHocMesh(
   buffer: ArrayBuffer;
   neighbors: Array<number>;
 }> {
-  const { position, additionalCoordinates, cubeSize, mappingName, scaleFactor, mag, ...rest } =
-    meshRequest;
+  const {
+    positionWithPadding,
+    additionalCoordinates,
+    cubeSize,
+    mappingName,
+    scaleFactor,
+    mag,
+    ...rest
+  } = meshRequest;
 
   return doWithToken(async (token) => {
     const params = new URLSearchParams();
@@ -1913,7 +1920,7 @@ export function computeAdHocMesh(
           // The back-end needs a small padding at the border of the
           // bounding box to calculate the mesh. This padding
           // is added here to the position and bbox size.
-          position: V3.toArray(V3.sub(position, mag)), // position is in mag1
+          position: positionWithPadding, // position is in mag1
           additionalCoordinates,
           cubeSize: V3.toArray(V3.add(cubeSize, [1, 1, 1])), //cubeSize is in target mag
           // Name and type of mapping to apply before building mesh (optional)
diff --git a/frontend/javascripts/types/api_types.ts b/frontend/javascripts/types/api_types.ts
@@ -1280,6 +1280,8 @@ export type FolderUpdater = {
 
 export enum CAMERA_POSITIONS {
   MOVING = "MOVING",
+  STATIC_ISOMETRIC = "STATIC_ISOMETRIC",
+  STATIC_XY = "STATIC_XY",
   STATIC_XZ = "STATIC_XZ",
   STATIC_YZ = "STATIC_YZ",
 }
diff --git a/frontend/javascripts/viewer/model/bucket_data_handling/bounding_box.ts b/frontend/javascripts/viewer/model/bucket_data_handling/bounding_box.ts
@@ -103,6 +103,10 @@ class BoundingBox {
     };
   }
 
+  clipPositionIntoBoundingBox(position: Vector3): Vector3 {
+    return V3.toArray(V3.max(this.min, V3.min(position, this.max)));
+  }
+
   extend(other: BoundingBox): BoundingBox {
     const newMin = V3.min(this.min, other.min);
     const newMax = V3.max(this.max, other.max);
diff --git a/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts
@@ -442,10 +442,9 @@ function* maybeLoadMeshChunk(
   threeDMap.set(clippedPosition, true);
   const scaleFactor = yield* select((state) => state.dataset.dataSource.scale.factor);
   const dataStoreHost = yield* select((state) => state.dataset.dataStore.url);
-  const owningOrganization = yield* select((state) => state.dataset.owningOrganization);
-  const datasetDirectoryName = yield* select((state) => state.dataset.directoryName);
+  const datasetId = yield* select((state) => state.dataset.id);
   const tracingStoreHost = yield* select((state) => state.annotation.tracingStore.url);
-  const dataStoreUrl = `${dataStoreHost}/data/datasets/${owningOrganization}/${datasetDirectoryName}/layers/${
+  const dataStoreUrl = `${dataStoreHost}/data/datasets/${datasetId}/layers/${
     layer.fallbackLayer != null ? layer.fallbackLayer : layer.name
   }`;
   const tracingStoreUrl = `${tracingStoreHost}/tracings/volume/${layer.name}`;
@@ -463,6 +462,9 @@ function* maybeLoadMeshChunk(
   const { segmentMeshController } = getSceneController();
 
   const cubeSize = marchingCubeSizeInTargetMag();
+  const paddedPosition = V3.toArray(V3.sub(clippedPosition, mag));
+  const paddedPositionWithinLayer =
+    layer.cube.boundingBox.clipPositionIntoBoundingBox(paddedPosition);
 
   while (retryCount < MAX_RETRY_COUNT) {
     try {
@@ -473,7 +475,7 @@ function* maybeLoadMeshChunk(
         },
         useDataStore ? dataStoreUrl : tracingStoreUrl,
         {
-          position: clippedPosition,
+          positionWithPadding: paddedPositionWithinLayer,
           additionalCoordinates: additionalCoordinates || undefined,
           mag,
           segmentId,
diff --git a/frontend/javascripts/viewer/view/action-bar/create_animation_modal.tsx b/frontend/javascripts/viewer/view/action-bar/create_animation_modal.tsx
@@ -406,12 +406,18 @@ function CreateAnimationModal(props: Props) {
                 <Radio.Button value={CAMERA_POSITIONS.MOVING}>
                   Camera circling around the dataset
                 </Radio.Button>
+                <Radio.Button value={CAMERA_POSITIONS.STATIC_XY}>
+                  Static camera looking at XY-viewport{" "}
+                </Radio.Button>
                 <Radio.Button value={CAMERA_POSITIONS.STATIC_XZ}>
                   Static camera looking at XZ-viewport{" "}
                 </Radio.Button>
                 <Radio.Button value={CAMERA_POSITIONS.STATIC_YZ}>
                   Static camera looking at YZ-viewport{" "}
                 </Radio.Button>
+                <Radio.Button value={CAMERA_POSITIONS.STATIC_ISOMETRIC}>
+                  Static camera with an isometric perspective looking at all 3 viewports{" "}
+                </Radio.Button>
               </Space>
             </Radio.Group>
           </Col>
diff --git a/frontend/javascripts/viewer/view/action_bar_view.tsx b/frontend/javascripts/viewer/view/action_bar_view.tsx
@@ -205,8 +205,16 @@ function CreateAnnotationButton() {
   const ButtonWithAuthentication = withAuthentication<AsyncButtonProps, typeof AsyncButton>(
     AsyncButton,
   );
+
   return (
-    <>
+    <div
+      onKeyDownCapture={(e: React.KeyboardEvent) => {
+        // Prevent closing the modal upon pressing some keys.
+        if (e.ctrlKey || e.metaKey || e.key === "AltGraph" || e.getModifierState("AltGraph")) {
+          e.stopPropagation();
+        }
+      }}
+    >
       <ButtonWithAuthentication
         activeUser={activeUser}
         authenticationMessage="You have to register or login to create an annotation."
@@ -231,7 +239,7 @@ function CreateAnnotationButton() {
           setSelectedSegmentationLayerName={setSelectedLayerName}
         />
       </Modal>
-    </>
+    </div>
   );
 }
 
diff --git a/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segment_list_item.tsx b/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segment_list_item.tsx
@@ -62,6 +62,7 @@ export function ColoredDotIcon({ colorRGBA }: { colorRGBA: Vector4 }) {
         alignSelf: "flex-start",
         marginTop: 5,
         marginLeft: 2,
+        flex: "0 0 10px",
       }}
     />
   );
diff --git a/project/BuildInfoSettings.scala b/project/BuildInfoSettings.scala
diff --git a/unreleased_changes/8888.md b/unreleased_changes/8888.md
diff --git a/unreleased_changes/8895.md b/unreleased_changes/8895.md
diff --git a/unreleased_changes/8901.md b/unreleased_changes/8901.md
diff --git a/unreleased_changes/8903.md b/unreleased_changes/8903.md
diff --git a/unreleased_changes/8909.md b/unreleased_changes/8909.md
diff --git a/unreleased_changes/8911.md b/unreleased_changes/8911.md
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
