Encourage coarser mags when annotating a high number of voxels

conf/application.conf

@@ -171,6 +171,7 @@ features {
   publicDemoDatasetUrl = "https://webknossos.org/datasets/scalable_minds/l4dense_motta_et_al_demo"
   exportTiffMaxVolumeMVx = 1024
   exportTiffMaxEdgeLengthVx = 8192
+  bucketSaveWarningThreshold = 1000
   defaultToLegacyBindings = false
   editableMappingsEnabled = true
   # The only valid item value is currently "ConnectomeView":

docs/volume_annotation/import_export.md

@@ -16,4 +16,20 @@ After finishing the annotation of a volume layer with a fallback layer, the comb
 Icon to open the materialize volume annotation dialog
 ///
 
-This button opens up a dialog that starts a long-running job which will materialize the volume annotation.
+This button opens up a dialog that starts a long-running job which will materialize the volume annotation.
+
+## Restricting magnifications
+
+WEBKNOSSOS allows data annotation in different magnifications.
+Restricting the available magnifications can greatly improve the performance when annotating large structures, such as nuclei, since the volume data does not need to be stored in all quality levels. 
+How to read: Mag 1 is the most detailed, 4-4-2 is downsampled by factor 4 in x and y, and by factor 2 in z.
+When annotating large structures, consider restricting the available magnifications in this specific annotation or in the volume annotation layer(s).
+The restriction can be defined when a new volume layer is created (either when an annotation is created or when a new volume layer is added to an annotation).
+
+### Restricting magnifications when creating an annotation
+
+Go to the dataset dashboard. Click on the three dots next to `New Annotation` for the dataset you want to annotate. Use the slider to restrict the volume magnifications, e.g. by omitting the finest magnification, which is usually 1-1-1, or the two finest magnifications.
+
+### Restricting magnifications when adding a new volume annotation layer
+
+Within an existing annotation, go to the `Layers` tab on the left. Click `Add Volume Annotation Layer` at the bottom of the list of all layers in this annotation. Restrict the magnifications as explained above.

frontend/javascripts/libs/toast.tsx

@@ -17,7 +17,9 @@ export type ToastConfig = {
   sticky?: boolean;
   timeout?: number;
   key?: string;
+  customFooter?: React.ReactNode;
   onClose?: () => void;
+  className?: string;
 };
 
 export type NotificationAPI = ReturnType<typeof notification.useNotification>[0];
@@ -148,8 +150,9 @@ const Toast = {
       duration: useManualTimeout || sticky ? 0 : timeOutInSeconds,
       message: toastMessage,
       style: {},
-      className: "",
+      className: config.className || "",
       onClose,
+      btn: config.customFooter,
     };
 
     if (type === "error") {

frontend/javascripts/libs/utils.ts

@@ -1404,3 +1404,13 @@ export async function retryAsyncFunction<T>(
     }
   }
 }
+
+/**
+ * Converts a string to a boolean value.
+ * Returns false for invalid inputs.
+ */
+export function stringToBoolean(value: string): boolean {
+  const normalized = value.trim().toLowerCase();
+  if (normalized === "true") return true;
+  return false;
+}

frontend/javascripts/test/backend-snapshot-tests/__snapshots__/misc.e2e.ts.snap

@@ -61,6 +61,7 @@ exports[`Misc APIs (E2E)  > getFeatureToggles() 1`] = `
 {
   "alignmentCostPerGVx": 0.5,
   "allowDeleteDatasets": true,
+  "bucketSaveWarningThreshold": 1000,
   "costPerCreditInDollar": 5.75,
   "costPerCreditInEuro": 5,
   "defaultToLegacyBindings": false,

frontend/javascripts/test/helpers/apiHelpers.ts

@@ -52,6 +52,7 @@ import {
 } from "test/fixtures/hybridtracing_server_objects";
 import type { ElementClass, ServerTracing } from "types/api_types";
 import { getConstructorForElementClass } from "viewer/model/helpers/typed_buffer";
+import { __setFeatures } from "features";
 
 const TOKEN = "secure-token";
 const ANNOTATION_TYPE = "annotationTypeValue";
@@ -377,6 +378,8 @@ export async function setupWebknossosForTesting(
     segmentMeshController: { meshesGroupsPerSegmentId: {} },
   });
 
+  __setFeatures({});
+
   try {
     await Model.fetch(
       null, // no compound annotation

frontend/javascripts/test/model/binary/layers/wkstore_adapter.spec.ts

@@ -312,7 +312,7 @@ describe("wkstore_adapter", () => {
 
     // @ts-ignore pushTransaction is a private method
     return pushQueue.pushTransaction(batch).then(() => {
-      expect(Store.dispatch).toHaveBeenCalledTimes(1);
+      expect(Store.dispatch).toHaveBeenCalledTimes(2);
       expect(Store.dispatch).toHaveBeenCalledWith(expectedSaveQueueItems);
     });
   });

frontend/javascripts/types/api_types.ts

@@ -760,6 +760,7 @@ export type APIFeatureToggles = {
   readonly publicDemoDatasetUrl: string;
   readonly exportTiffMaxVolumeMVx: number;
   readonly exportTiffMaxEdgeLengthVx: number;
+  readonly bucketSaveWarningThreshold: number;
   readonly defaultToLegacyBindings: boolean;
   readonly editableMappingsEnabled?: boolean;
   readonly optInTabs?: Array<string>;

frontend/javascripts/viewer/model/actions/annotation_actions.ts

@@ -77,6 +77,9 @@ export type AddPrecomputedMeshAction = ReturnType<typeof addPrecomputedMeshActio
 export type SetOthersMayEditForAnnotationAction = ReturnType<
   typeof setOthersMayEditForAnnotationAction
 >;
+export type ShowManyBucketUpdatesWarningAction = ReturnType<
+  typeof showManyBucketUpdatesWarningAction
+>;
 
 export type AnnotationActionTypes =
   | InitializeAnnotationAction
@@ -373,6 +376,11 @@ export const setOthersMayEditForAnnotationAction = (othersMayEdit: boolean) =>
     othersMayEdit,
   }) as const;
 
+export const showManyBucketUpdatesWarningAction = () =>
+  ({
+    type: "SHOW_MANY_BUCKET_UPDATES_WARNING",
+  }) as const;
+
 export const dispatchMaybeFetchMeshFilesAsync = async (
   dispatch: Dispatch<any>,
   segmentationLayer: APIDataLayer | null | undefined,

frontend/javascripts/viewer/model/actions/save_actions.ts

@@ -14,6 +14,7 @@ export type PushSaveQueueTransaction = {
   items: UpdateAction[];
   transactionId: string;
 };
+export type NotifyAboutUpdatedBucketsAction = ReturnType<typeof notifyAboutUpdatedBucketsAction>;
 type SaveNowAction = ReturnType<typeof saveNowAction>;
 export type ShiftSaveQueueAction = ReturnType<typeof shiftSaveQueueAction>;
 type DiscardSaveQueuesAction = ReturnType<typeof discardSaveQueuesAction>;
@@ -31,6 +32,7 @@ export type SaveAction =
   | PushSaveQueueTransaction
   | SaveNowAction
   | ShiftSaveQueueAction
+  | NotifyAboutUpdatedBucketsAction
   | DiscardSaveQueuesAction
   | SetSaveBusyAction
   | SetLastSaveTimestampAction
@@ -61,6 +63,9 @@ export const pushSaveQueueTransactionIsolated = (
     transactionId: getUid(),
   }) as const;
 
+export const notifyAboutUpdatedBucketsAction = (count: number) =>
+  ({ type: "NOTIFY_ABOUT_UPDATED_BUCKETS", count }) as const;
+
 export const saveNowAction = () =>
   ({
     type: "SAVE_NOW",

frontend/javascripts/viewer/model/bucket_data_handling/pushqueue.ts

@@ -6,7 +6,7 @@ import type DataCube from "viewer/model/bucket_data_handling/data_cube";
 import { createCompressedUpdateBucketActions } from "viewer/model/bucket_data_handling/wkstore_adapter";
 import Store from "viewer/store";
 import { escalateErrorAction } from "../actions/actions";
-import { pushSaveQueueTransaction } from "../actions/save_actions";
+import { notifyAboutUpdatedBucketsAction, pushSaveQueueTransaction } from "../actions/save_actions";
 import type { UpdateActionWithoutIsolationRequirement } from "../sagas/volume/update_actions";
 
 // Only process the PushQueue after there was no user interaction (or bucket modification due to
@@ -155,7 +155,8 @@ class PushQueue {
         createCompressedUpdateBucketActions(batch),
       );
       Store.dispatch(pushSaveQueueTransaction(items));
-
+      Store.dispatch(notifyAboutUpdatedBucketsAction(items.length));
+      console.log("notify about ", items.length, " items");
       this.compressingBucketCount -= batch.length;
     } catch (error) {
       // See other usage of escalateErrorAction for a detailed explanation.

frontend/javascripts/viewer/model/sagas/many_bucket_updates_warning_saga.tsx

@@ -0,0 +1,87 @@
+import { Button, Checkbox, type CheckboxChangeEvent, Space } from "antd";
+import Toast from "libs/toast";
+import UserLocalStorage from "libs/user_local_storage";
+import { stringToBoolean } from "libs/utils";
+import { takeEvery } from "typed-redux-saga";
+import type { Saga } from "viewer/model/sagas/effect-generators";
+
+const TOO_MANY_BUCKETS_TOAST_KEY = "manyBucketUpdatesWarningToast";
+const WARNING_SUPPRESSION_USER_STORAGE_KEY = "suppressManyBucketUpdatesWarning";
+
+function* manyBucketUpdatesWarning(): Saga<void> {
+  let showWarningToastInThisSession = true;
+  const setShowWarningToastInThisSession = (value: boolean) => {
+    showWarningToastInThisSession = value;
+  };
+  setInterval(() => {
+    UserLocalStorage.setItem(WARNING_SUPPRESSION_USER_STORAGE_KEY, "false");
+    console.log("resetting suppressBucketWarning to false every 120s for dev purposes");
+  }, 120 * 1000);
+  //TODO_C dev
+
+  function* showWarningToast(): Saga<void> {
+    let neverShowAgain = false;
+
+    const onClose = () => {
+      Toast.notificationAPI?.destroy(TOO_MANY_BUCKETS_TOAST_KEY);
+      if (neverShowAgain) {
+        UserLocalStorage.setItem(WARNING_SUPPRESSION_USER_STORAGE_KEY, neverShowAgain.toString());
+      }
+    };
+    const handleCheckboxChange = (event: CheckboxChangeEvent) => {
+      neverShowAgain = event.target.checked;
+    };
+
+    const warningMessage =
+      "You are annotating a large area with fine magnifications. This can significantly slow down WEBKNOSSOS. Consider creating an annotation or annotation layer with restricted magnifications.";
+    const linkToDocs =
+      "https://docs.webknossos.org/volume_annotation/import_export.html#restricting-magnifications";
+    const neverShowAgainCheckbox = (
+      <Checkbox onChange={handleCheckboxChange} style={{ marginTop: "8px", marginBottom: "5px" }}>
+        Never show this again
+      </Checkbox>
+    );
+    const closeButton = <Button onClick={onClose}>Close</Button>;
+    const linkToDocsButton = (
+      <Button href={linkToDocs} target="_blank" rel="noopener noreferrer" type="primary">
+        Learn how
+      </Button>
+    );
+    const footer = (
+      <Space>
+        {linkToDocsButton}
+        {closeButton}
+      </Space>
+    );
+
+    const suppressManyBucketUpdatesWarning = stringToBoolean(
+      UserLocalStorage.getItem(WARNING_SUPPRESSION_USER_STORAGE_KEY) || "false",
+    );
+
+    if (showWarningToastInThisSession && !suppressManyBucketUpdatesWarning) {
+      console.warn(warningMessage + " For more info, visit: " + linkToDocs);
+      Toast.warning(
+        <>
+          {warningMessage}
+          <br />
+          {neverShowAgainCheckbox}
+        </>,
+        {
+          customFooter: footer,
+          key: TOO_MANY_BUCKETS_TOAST_KEY,
+          sticky: true,
+          onClose,
+          className: "many-bucket-updates-warning",
+        },
+      );
+      setShowWarningToastInThisSession(false);
+    } else {
+      console.log("suppressing warning toast"); //TODO_C dev
+    }
+  }
+  yield takeEvery("SHOW_MANY_BUCKET_UPDATES_WARNING", showWarningToast);
+}
+
+export default function* manyBucketUpdatesWarningSaga(): Saga<void> {
+  yield takeEvery("WK_READY", manyBucketUpdatesWarning);
+}

frontend/javascripts/viewer/model/sagas/root_saga.ts

@@ -21,6 +21,7 @@ import VolumetracingSagas from "viewer/model/sagas/volumetracing_saga";
 import type { EscalateErrorAction } from "../actions/actions";
 import { setIsWkReadyAction } from "../actions/ui_actions";
 import maintainMaximumZoomForAllMagsSaga from "./flycam_info_cache_saga";
+import manyBucketUpdatesWarningSaga from "./many_bucket_updates_warning_saga";
 import adHocMeshSaga from "./meshes/ad_hoc_mesh_saga";
 import commonMeshSaga, { handleAdditionalCoordinateUpdate } from "./meshes/common_mesh_saga";
 import precomputedMeshSaga from "./meshes/precomputed_mesh_saga";
@@ -88,6 +89,7 @@ function* restartableSaga(): Saga<void> {
       ...DatasetSagas.map((saga) => call(saga)),
       call(splitBoundaryMeshSaga),
       call(toolSaga),
+      call(manyBucketUpdatesWarningSaga),
     ]);
   } catch (err: any) {
     rootSagaCrashed = true;

frontend/javascripts/viewer/model/sagas/saving/save_saga.ts → frontend/javascripts/viewer/model/sagas/saving/save_saga.tsx

@@ -1,19 +1,24 @@
 import { getUpdateActionLog } from "admin/rest_api";
+import features from "features";
 import ErrorHandling from "libs/error_handling";
 import Toast from "libs/toast";
 import { sleep } from "libs/utils";
 import _ from "lodash";
-import { call, fork, put, takeEvery } from "typed-redux-saga";
+import { call, delay, fork, put, takeEvery } from "typed-redux-saga";
 import type { APIUpdateActionBatch } from "types/api_types";
 import { getLayerByName, getMappingInfo } from "viewer/model/accessors/dataset_accessor";
-import { setVersionNumberAction } from "viewer/model/actions/save_actions";
+import { showManyBucketUpdatesWarningAction } from "viewer/model/actions/annotation_actions";
+import {
+  type NotifyAboutUpdatedBucketsAction,
+  setVersionNumberAction,
+} from "viewer/model/actions/save_actions";
 import { applySkeletonUpdateActionsFromServerAction } from "viewer/model/actions/skeletontracing_actions";
 import { applyVolumeUpdateActionsFromServerAction } from "viewer/model/actions/volumetracing_actions";
 import { globalPositionToBucketPositionWithMag } from "viewer/model/helpers/position_converter";
 import type { Saga } from "viewer/model/sagas/effect-generators";
 import { select } from "viewer/model/sagas/effect-generators";
 import { ensureWkReady } from "viewer/model/sagas/ready_sagas";
-import { Model } from "viewer/singletons";
+import { Model, Store } from "viewer/singletons";
 import type { SkeletonTracing, VolumeTracing } from "viewer/store";
 import { takeEveryWithBatchActionSupport } from "../saga_helpers";
 import { updateLocalHdf5Mapping } from "../volume/mapping_saga";
@@ -31,11 +36,48 @@ export function* setupSavingToServer(): Saga<void> {
   yield* takeEvery("INITIALIZE_ANNOTATION_WITH_TRACINGS", setupSavingForAnnotation);
   yield* takeEveryWithBatchActionSupport("INITIALIZE_SKELETONTRACING", setupSavingForTracingType);
   yield* takeEveryWithBatchActionSupport("INITIALIZE_VOLUMETRACING", setupSavingForTracingType);
+  yield* takeEvery("WK_READY", watchForNumberOfBucketsInSaveQueue);
 }
 
 const VERSION_POLL_INTERVAL_COLLAB = 10 * 1000;
 const VERSION_POLL_INTERVAL_READ_ONLY = 60 * 1000;
 const VERSION_POLL_INTERVAL_SINGLE_EDITOR = 30 * 1000;
+// interval at which the number of buckets in save queue is checked
+const CHECK_NUMBER_OF_BUCKETS_IN_SAVE_QUEUE_INTERVAL_MS = 10 * 1000;
+// sliding time window for which the number of buckets in save queue is summed up
+const CHECK_NUMBER_OF_BUCKETS_SLIDING_WINDOW_MS = 120 * 1000;
+
+function* watchForNumberOfBucketsInSaveQueue(): Saga<void> {
+  const bucketSaveWarningThreshold = features().bucketSaveWarningThreshold;
+  let bucketsForCurrentInterval = 0;
+  let currentBucketCounts: Array<number> = [];
+  const bucketCountArrayLength = Math.floor(
+    CHECK_NUMBER_OF_BUCKETS_SLIDING_WINDOW_MS / CHECK_NUMBER_OF_BUCKETS_IN_SAVE_QUEUE_INTERVAL_MS,
+  );
+  yield* takeEvery("NOTIFY_ABOUT_UPDATED_BUCKETS", (action: NotifyAboutUpdatedBucketsAction) => {
+    bucketsForCurrentInterval += action.count;
+  });
+  while (true) {
+    yield* delay(CHECK_NUMBER_OF_BUCKETS_IN_SAVE_QUEUE_INTERVAL_MS);
+    const sumOfBuckets = _.sum(currentBucketCounts);
+    console.log(
+      "buckets in last interval: ",
+      bucketsForCurrentInterval,
+      "currentBucketsArray: ",
+      currentBucketCounts,
+      "sumOfBuckets: ",
+      sumOfBuckets,
+    );
+    if (sumOfBuckets > bucketSaveWarningThreshold) {
+      Store.dispatch(showManyBucketUpdatesWarningAction());
+    }
+    currentBucketCounts.push(bucketsForCurrentInterval);
+    if (currentBucketCounts.length > bucketCountArrayLength) {
+      currentBucketCounts.shift();
+    }
+    bucketsForCurrentInterval = 0;
+  }
+}
 
 function* watchForSaveConflicts(): Saga<void> {
   function* checkForNewVersion(): Saga<boolean> {

frontend/stylesheets/main.less

@@ -716,4 +716,10 @@ button.narrow {
   >div:last-of-type {
     margin-bottom: 15px;
   }
+}
+
+.many-bucket-updates-warning{
+  .ant-notification-notice-btn{
+    margin-top: 0 !important;
+  }
 }