Guard against failing agglomerate requests (#8767)

### URL of deployed dev instance (used for testing):
- https://___.webknossos.xyz

### Steps to test:
- create an annotation for a ds with an agglomerate file
- enable a hdf5 mapping
- switch to proofreading tool
- every second request should fail (see network tab)
- the user shouldn't notice this (except for the delay)
- set `window.alwaysFail = true` <-- every request should fail <-- the
mapping saga should not crash

### Issues:
- contributes to #8715 

------
(Please delete unneeded items, merge only when none are left open)
- [x] Added changelog entry (create a `$PR_NUMBER.md` file in
`unreleased_changes` or use `./tools/create-changelog-entry.py`)
- [ ] Added migration guide entry if applicable (edit the same file as
for the changelog)
- [ ] Updated [documentation](../blob/master/docs) if applicable
- [ ] Adapted [wk-libs python
client](https://github.com/scalableminds/webknossos-libs/tree/master/webknossos/webknossos/client)
if relevant API parts change
- [ ] Removed dev-only changes like prints and application.conf edits
- [ ] Considered [common edge
cases](../blob/master/.github/common_edge_cases.md)
- [ ] Needs datastore update after deployment

Author: philippotto

frontend/javascripts/admin/rest_api.ts

@@ -2001,15 +2001,17 @@ export async function getAgglomeratesForSegmentsFromDatastore<T extends number |
   const segmentIdBuffer = serializeProtoListOfLong<T>(segmentIds);
   const listArrayBuffer: ArrayBuffer = await doWithToken((token) => {
     const params = new URLSearchParams({ token });
-    return Request.receiveArraybuffer(
-      `${dataStoreUrl}/data/datasets/${dataSourceId.owningOrganization}/${dataSourceId.directoryName}/layers/${layerName}/agglomerates/${mappingId}/agglomeratesForSegments?${params}`,
-      {
-        method: "POST",
-        body: segmentIdBuffer,
-        headers: {
-          "Content-Type": "application/octet-stream",
+    return Utils.retryAsyncFunction(() =>
+      Request.receiveArraybuffer(
+        `${dataStoreUrl}/data/datasets/${dataSourceId.owningOrganization}/${dataSourceId.directoryName}/layers/${layerName}/agglomerates/${mappingId}/agglomeratesForSegments?${params}`,
+        {
+          method: "POST",
+          body: segmentIdBuffer,
+          headers: {
+            "Content-Type": "application/octet-stream",
+          },
         },
-      },
+      ),
     );
   });
   // Ensure that the values are bigint if the keys are bigint
@@ -2041,15 +2043,18 @@ export async function getAgglomeratesForSegmentsFromTracingstore<T extends numbe
   );
   const listArrayBuffer: ArrayBuffer = await doWithToken((token) => {
     params.set("token", token);
-    return Request.receiveArraybuffer(
-      `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomeratesForSegments?${params}`,
-      {
-        method: "POST",
-        body: segmentIdBuffer,
-        headers: {
-          "Content-Type": "application/octet-stream",
+    return Utils.retryAsyncFunction(() =>
+      Request.receiveArraybuffer(
+        `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomeratesForSegments?${params}`,
+        {
+          method: "POST",
+          body: segmentIdBuffer,
+          headers: {
+            "Content-Type": "application/octet-stream",
+          },
+          showErrorToast: false,
         },
-      },
+      ),
     );
   });
 
@@ -2231,17 +2236,19 @@ export async function getEdgesForAgglomerateMinCut(
   },
 ): Promise<Array<MinCutTargetEdge>> {
   return doWithToken((token) =>
-    Request.sendJSONReceiveJSON(
-      `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomerateGraphMinCut?token=${token}`,
-      {
-        data: {
-          ...segmentsInfo,
-          // TODO: Proper 64 bit support (#6921)
-          segmentId1: Number(segmentsInfo.segmentId1),
-          segmentId2: Number(segmentsInfo.segmentId2),
-          agglomerateId: Number(segmentsInfo.agglomerateId),
+    Utils.retryAsyncFunction(() =>
+      Request.sendJSONReceiveJSON(
+        `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomerateGraphMinCut?token=${token}`,
+        {
+          data: {
+            ...segmentsInfo,
+            // TODO: Proper 64 bit support (#6921)
+            segmentId1: Number(segmentsInfo.segmentId1),
+            segmentId2: Number(segmentsInfo.segmentId2),
+            agglomerateId: Number(segmentsInfo.agglomerateId),
+          },
         },
-      },
+      ),
     ),
   );
 }
@@ -2262,16 +2269,18 @@ export async function getNeighborsForAgglomerateNode(
   },
 ): Promise<NeighborInfo> {
   return doWithToken((token) =>
-    Request.sendJSONReceiveJSON(
-      `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomerateGraphNeighbors?token=${token}`,
-      {
-        data: {
-          ...segmentInfo,
-          // TODO: Proper 64 bit support (#6921)
-          segmentId: Number(segmentInfo.segmentId),
-          agglomerateId: Number(segmentInfo.agglomerateId),
+    Utils.retryAsyncFunction(() =>
+      Request.sendJSONReceiveJSON(
+        `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomerateGraphNeighbors?token=${token}`,
+        {
+          data: {
+            ...segmentInfo,
+            // TODO: Proper 64 bit support (#6921)
+            segmentId: Number(segmentInfo.segmentId),
+            agglomerateId: Number(segmentInfo.agglomerateId),
+          },
         },
-      },
+      ),
     ),
   );
 }

frontend/javascripts/components/error_boundary.tsx

@@ -52,7 +52,7 @@ export default class ErrorBoundary extends React.Component<
               <a href="#" onClick={this.clearLocalStorageAndReload}>
                 here
               </a>{" "}
-              to do so.
+              to do so. If this does not help, please clear the browser cache.
             </Typography.Paragraph>
             <Alert
               style={{ maxHeight: "70vh", overflow: "auto" }}

frontend/javascripts/libs/utils.ts

@@ -1391,3 +1391,24 @@ export const ColoredLogger = {
     console.log(chalk.bgBlue(str), ...args);
   },
 };
+
+export async function retryAsyncFunction<T>(
+  fn: () => Promise<T>,
+  retryCount: number = 3, // 0 retries would only do 1 attempt
+  exponentialDelayFactor: number = 1000,
+): Promise<T> {
+  let currentAttempt = 0;
+  while (true) {
+    try {
+      return await fn();
+    } catch (exception) {
+      if (currentAttempt === retryCount) {
+        throw exception;
+      } else {
+        console.warn("Failed to run async function due to", exception, ". Will retry now.");
+      }
+      currentAttempt++;
+      await sleep(exponentialDelayFactor * 2 ** currentAttempt);
+    }
+  }
+}

frontend/javascripts/viewer/model/sagas/volume/mapping_saga.ts

@@ -477,24 +477,33 @@ export function* updateLocalHdf5Mapping(
     intersection: mutableRemainingEntries,
   } = fastDiffSetAndMap(segmentIds as Set<NumberLike>, previousMapping);
 
-  const newEntries =
-    editableMapping != null
-      ? yield* call(
-          getAgglomeratesForSegmentsFromTracingstore,
-          annotation.tracingStore.url,
-          editableMapping.tracingId,
-          Array.from(newSegmentIds),
-          annotation.annotationId,
-          annotation.version,
-        )
-      : yield* call(
-          getAgglomeratesForSegmentsFromDatastore,
-          dataset.dataStore.url,
-          dataset,
-          mappingLayerName,
-          mappingName,
-          Array.from(newSegmentIds),
-        );
+  let newEntries;
+  try {
+    newEntries =
+      editableMapping != null
+        ? yield* call(
+            getAgglomeratesForSegmentsFromTracingstore,
+            annotation.tracingStore.url,
+            editableMapping.tracingId,
+            Array.from(newSegmentIds),
+            annotation.annotationId,
+            annotation.version,
+          )
+        : yield* call(
+            getAgglomeratesForSegmentsFromDatastore,
+            dataset.dataStore.url,
+            dataset,
+            mappingLayerName,
+            mappingName,
+            Array.from(newSegmentIds),
+          );
+  } catch (exception) {
+    console.error("Could not load agglomerate ids for segments due to", exception);
+    Toast.error(
+      "Could not load agglomerate ids for segments. Some segments will remain hidden for now.",
+    );
+    return;
+  }
 
   // It is safe to mutate mutableRemainingEntries to compute the merged,
   // new mapping. See the definition of mutableRemainingEntries.

frontend/javascripts/viewer/model/sagas/volume/proofread_saga.ts

@@ -533,12 +533,19 @@ function* performMinCut(
     editableMappingId: volumeTracingId,
   };
 
-  const edgesToRemove = yield* call(
-    getEdgesForAgglomerateMinCut,
-    tracingStoreUrl,
-    volumeTracingId,
-    segmentsInfo,
-  );
+  let edgesToRemove;
+  try {
+    edgesToRemove = yield* call(
+      getEdgesForAgglomerateMinCut,
+      tracingStoreUrl,
+      volumeTracingId,
+      segmentsInfo,
+    );
+  } catch (exception) {
+    console.error(exception);
+    Toast.error("Could not determine which edges to delete for cut. Please try again.");
+    return true;
+  }
 
   // Use untransformedPosition below because agglomerate trees should not have
   // any transforms, anyway.
@@ -598,12 +605,19 @@ function* performCutFromNeighbors(
     editableMappingId: volumeTracingId,
   };
 
-  const neighborInfo = yield* call(
-    getNeighborsForAgglomerateNode,
-    tracingStoreUrl,
-    volumeTracingId,
-    segmentsInfo,
-  );
+  let neighborInfo;
+  try {
+    neighborInfo = yield* call(
+      getNeighborsForAgglomerateNode,
+      tracingStoreUrl,
+      volumeTracingId,
+      segmentsInfo,
+    );
+  } catch (exception) {
+    console.error(exception);
+    Toast.error("Could not load neighbors of agglomerate node. Please try again.");
+    return { didCancel: true };
+  }
 
   const edgesToRemove: Array<
     | {

unreleased_changes/8767.md

@@ -0,0 +1,2 @@
+### Fixed
+- Fixed that WEBKNOSSOS would crash directly if the communication with the server was interrupted while using the proofreading tool.