fix: LSDV-4711: CORS error accessing previously annotated data (#1299)

* fix: LSDV-4711: CORS error accessing previously annotated data

* allow audio v3 to resetSource of a previously working url as the presigned url may have expired

* retry paragraph audio source if it has previously loaded but encountered an error, as this may be a presigned url requiring refresh

* retry Video if the source previously loaded but now is in error, could just require refresh of the presigned url

Author: bmartel

src/components/ImageView/Image.js

@@ -1,6 +1,7 @@
 import { observer } from 'mobx-react';
 import { forwardRef, useCallback, useMemo } from 'react';
 import { Block, Elem } from '../../utils/bem';
+import { FF_LSDV_4711, isFF } from '../../utils/feature-flags';
 import messages from '../../utils/messages';
 import { ErrorMessage } from '../ErrorMessage/ErrorMessage';
 import './Image.styl';
@@ -64,6 +65,10 @@ const ImageProgress = observer(({
   ) : null;
 });
 
+const imgDefaultProps = {};
+
+if (isFF(FF_LSDV_4711)) imgDefaultProps.crossOrigin = 'anonymous';
+
 const ImageRenderer = observer(forwardRef(({
   src,
   onLoad,
@@ -78,6 +83,7 @@ const ImageRenderer = observer(forwardRef(({
 
   return (
     <img
+      {...imgDefaultProps}
       ref={ref}
       alt="image"
       src={src}

src/components/ImageView/ImageView.js

@@ -20,13 +20,16 @@ import ResizeObserver from '../../utils/resize-observer';
 import { debounce } from '../../utils/debounce';
 import Constants from '../../core/Constants';
 import { fixRectToFit } from '../../utils/image';
-import { FF_DEV_1285, FF_DEV_1442, FF_DEV_3077, FF_DEV_3793, FF_DEV_4081, FF_LSDV_4583, FF_LSDV_4583_6, isFF } from '../../utils/feature-flags';
+import { FF_DEV_1285, FF_DEV_1442, FF_DEV_3077, FF_DEV_3793, FF_DEV_4081, FF_LSDV_4583, FF_LSDV_4583_6, FF_LSDV_4711, isFF } from '../../utils/feature-flags';
 import { Pagination } from '../../common/Pagination/Pagination';
 import { Image } from './Image';
 
 Konva.showWarnings = false;
 
 const hotkeys = Hotkey('Image');
+const imgDefaultProps = {};
+
+if (isFF(FF_LSDV_4711)) imgDefaultProps.crossOrigin = 'anonymous';
 
 const splitRegions = (regions) => {
   const brushRegions = [];
@@ -1080,6 +1083,7 @@ export default observer(
             <div className={styles.gallery}>
               {item.images.map((src, i) => (
                 <img
+                  {...imgDefaultProps}
                   alt=""
                   key={src}
                   src={src}

src/components/VideoCanvas/VideoCanvas.tsx

@@ -1,5 +1,6 @@
 import { forwardRef, memo, MutableRefObject, useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { Block, Elem } from '../../utils/bem';
+import { FF_LSDV_4711, isFF } from '../../utils/feature-flags';
 import { clamp, isDefined } from '../../utils/utilities';
 import './VideoCanvas.styl';
 import { MAX_ZOOM, MIN_ZOOM } from './VideoConstants';
@@ -35,6 +36,7 @@ type VideoProps = {
   onFrameChange?: (frame: number, length: number) => void,
   onEnded?: () => void,
   onResize?: (dimensions: VideoDimentions) => void,
+  onError?: (error: any) => void,
 }
 
 type PanOptions = {
@@ -92,6 +94,7 @@ export const VideoCanvas = memo(forwardRef<VideoRef, VideoProps>((props, ref) =>
   const contextRef = useRef<CanvasRenderingContext2D | null>();
   const videoRef = useRef<HTMLVideoElement>();
   const supportedFileTypeRef = useRef<boolean|null>(null);
+  const hasLoadedRef = useRef<boolean>(false);
 
   const canvasWidth = useMemo(() => props.width ?? 600, [props.width]);
   const canvasHeight = useMemo(() => props.height ?? 600, [props.height]);
@@ -187,6 +190,7 @@ export const VideoCanvas = memo(forwardRef<VideoRef, VideoProps>((props, ref) =>
       if (!playing) updateFrame(true);
 
       if (video.networkState === video.NETWORK_IDLE) {
+        hasLoadedRef.current = true;
         setBuffering(false);
       } else {
         setBuffering(true);
@@ -224,6 +228,24 @@ export const VideoCanvas = memo(forwardRef<VideoRef, VideoProps>((props, ref) =>
     props.onPause?.();
   }, [props.onEnded]);
 
+  const handleVideoError = useCallback(() => {
+    if (!isFF(FF_LSDV_4711)) return;
+
+    const video = videoRef.current;
+
+    if (video?.error && hasLoadedRef.current) {
+      hasLoadedRef.current = false;
+
+      // If the video errored after loading, we can try to reload it
+      // as it may have been a temporary network issue or signed url that expired
+      video.load();
+    } else if (video) {
+      // If the video never loaded and errored, we can't do anything about it
+      // so report it to the consumer
+      props.onError?.(video.error);
+    }
+  }, [props.onError]);
+
   const handleAnimationFrame = () => {
     updateFrame();
 
@@ -551,6 +573,7 @@ export const VideoCanvas = memo(forwardRef<VideoRef, VideoProps>((props, ref) =>
         onPlaying={handleVideoPlaying}
         onWaiting={handleVideoWaiting}
         onEnded={handleVideoEnded}
+        onError={handleVideoError}
       />
     </Block>
   );

src/components/VideoCanvas/VirtualVideo.tsx

@@ -1,5 +1,6 @@
 import { DetailedHTMLProps, forwardRef, useCallback, useEffect, useRef, VideoHTMLAttributes } from 'react';
 import InfoModal from '../../components/Infomodal/Infomodal';
+import { FF_LSDV_4711, isFF } from '../../utils/feature-flags';
 
 
 type VirtualVideoProps = DetailedHTMLProps<VideoHTMLAttributes<HTMLVideoElement>, HTMLVideoElement> & {
@@ -91,6 +92,8 @@ export const VirtualVideo = forwardRef<HTMLVideoElement, VirtualVideoProps>((pro
     videoEl.controls = false;
     videoEl.preload = 'auto';
 
+    if (isFF(FF_LSDV_4711)) videoEl.crossOrigin = 'anonymous';
+
     Object.assign(videoEl.style, {
       top: '-9999px',
       width: 0,

src/lib/AudioUltra/Controls/Player.ts

@@ -130,6 +130,7 @@ export class Player extends Destructable {
   init(audio: WaveformAudio) {
     this.audio = audio;
     this.audio.on('canplay', this.handleCanPlay);
+    this.audio.on('resetSource', this.handleResetSource);
     if (this.audio.el) {
       this.audio.el.addEventListener('play', this.handlePlayed);
       this.audio.el.addEventListener('pause', this.handlePaused);
@@ -180,6 +181,18 @@ export class Player extends Destructable {
     this.updateCurrentTime(true);
   };
 
+  private handleResetSource = async () => {
+    if (!this.audio?.el) return;
+
+    const wasPlaying = this.playing;
+
+    this.stop();
+    this.audio.el.load();
+
+    if (wasPlaying) this.play();
+  };
+
+
   private handleCanPlay = () => {
     this.bufferResolve?.();
   };

src/lib/AudioUltra/Media/WaveformAudio.ts

@@ -1,3 +1,4 @@
+import { FF_LSDV_4711, isFF } from '../../../utils/feature-flags';
 import { Events } from '../Common/Events';
 import { __DEBUG__ } from '../Common/Utils';
 import { audioDecoderPool } from './AudioDecoderPool';
@@ -15,6 +16,7 @@ export interface WaveformAudioOptions {
 interface WaveformAudioEvents {
   decodingProgress: (chunk: number, total: number) => void;
   canplay: () => void;
+  resetSource: () => void;
 }
 
 export class WaveformAudio extends Events<WaveformAudioEvents> {
@@ -33,6 +35,7 @@ export class WaveformAudio extends Events<WaveformAudioEvents> {
   private decoderType: 'ffmpeg' | 'webaudio' = 'ffmpeg';
   private src?: string;
   private mediaResolve?: () => void;
+  private hasLoadedSource = false;
 
   constructor(options: WaveformAudioOptions) {
     super();
@@ -182,6 +185,9 @@ export class WaveformAudio extends Events<WaveformAudioEvents> {
     this.el.preload = 'auto';
     this.el.setAttribute('data-testid', 'waveform-audio');
     this.el.style.display = 'none';
+
+    if (isFF(FF_LSDV_4711)) this.el.crossOrigin = 'anonymous';
+
     document.body.appendChild(this.el);
 
     this.mediaPromise = new Promise((resolve, reject) => {
@@ -190,22 +196,31 @@ export class WaveformAudio extends Events<WaveformAudioEvents> {
     });
 
     this.el.addEventListener('canplaythrough', this.mediaReady);
-    this.el.addEventListener('error', this.mediaReady);
+    this.el.addEventListener('error', this.mediaError);
     this.loadMedia();
   }
 
-  mediaReady = async (e: any) => {
-    if (e.type === 'error') {
-      this.mediaReject?.(this.el?.error);
+  mediaError = () => {
+    // If this source has already loaded, we will retry the source url
+    if (isFF(FF_LSDV_4711) && this.hasLoadedSource && this.el) {
+      this.hasLoadedSource = false;
+      this.invoke('resetSource');
     } else {
-      if (this.mediaResolve) {
-        this.mediaResolve?.();
-        this.mediaResolve = undefined;
-      }
-      this.invoke('canplay');
+      // otherwise it's an unrecoverable error
+      this.mediaReject?.(this.el?.error);
     }
   };
 
+  mediaReady = () => {
+    if (this.mediaResolve) {
+      this.mediaResolve?.();
+      this.mediaResolve = undefined;
+    }
+
+    this.hasLoadedSource = true;
+    this.invoke('canplay');
+  };
+
   /**
    * Load the media element with the audio source and begin an initial playback buffer
    */

src/tags/object/Image/Image.js

@@ -14,7 +14,7 @@ import { RectRegionModel } from '../../../regions/RectRegion';
 import * as Tools from '../../../tools';
 import ToolsManager from '../../../tools/Manager';
 import { parseValue } from '../../../utils/data';
-import { FF_DEV_3377, FF_DEV_3666, FF_DEV_3793, FF_DEV_4081, FF_LSDV_4583, FF_LSDV_4583_6, isFF } from '../../../utils/feature-flags';
+import { FF_DEV_3377, FF_DEV_3666, FF_DEV_3793, FF_DEV_4081, FF_LSDV_4583, FF_LSDV_4583_6, FF_LSDV_4711, isFF } from '../../../utils/feature-flags';
 import { guidGenerator } from '../../../utils/unique';
 import { clamp, isDefined } from '../../../utils/utilities';
 import ObjectBase from '../Base';
@@ -282,6 +282,8 @@ const Model = types.model({
   get imageCrossOrigin() {
     const value = self.crossorigin.toLowerCase();
 
+    if (isFF(FF_LSDV_4711) && (!value || value === 'none')) return 'anonymous';
+
     if (!isFF(FF_DEV_4081)) {
       return null;
     } else if (!value || value === 'none') {

src/tags/object/Paragraphs/HtxParagraphs.js

@@ -2,13 +2,17 @@ import React, { Component } from 'react';
 import { inject, observer } from 'mobx-react';
 
 import ObjectTag from '../../../components/Tags/Object';
-import { FF_DEV_2669, FF_DEV_2918, FF_LSDV_3012, isFF } from '../../../utils/feature-flags';
+import { FF_DEV_2669, FF_DEV_2918, FF_LSDV_3012, FF_LSDV_4711, isFF } from '../../../utils/feature-flags';
 import { findNodeAt, matchesSelector, splitBoundaries } from '../../../utils/html';
 import { isSelectionContainsSpan } from '../../../utils/selection-tools';
 import styles from './Paragraphs.module.scss';
 import { AuthorFilter } from './AuthorFilter';
 import { Phrases } from './Phrases';
 
+const audioDefaultProps = {};
+
+if (isFF(FF_LSDV_4711)) audioDefaultProps.crossOrigin = 'anonymous';
+
 class HtxParagraphsView extends Component {
   _regionSpanSelector = '.htx-highlight';
 
@@ -414,6 +418,7 @@ class HtxParagraphsView extends Component {
       <ObjectTag item={item} className={'lsf-paragraphs'}>
         {withAudio && (
           <audio
+            {...audioDefaultProps} 
             controls={item.showplayer && !item.syncedAudio}
             className={styles.audio}
             src={item.audio}
@@ -424,6 +429,8 @@ class HtxParagraphsView extends Component {
               ref: item.getRef(),
             })}
             onEnded={item.reset}
+            onError={item.handleError}
+            onCanPlay={item.handleCanPlay}
           />
         )}
         {isFF(FF_DEV_2669) && <AuthorFilter item={item} />}

src/tags/object/Paragraphs/model.js

@@ -10,13 +10,14 @@ import { SyncMixin } from '../../../mixins/SyncMixin';
 import { ParagraphsRegionModel } from '../../../regions/ParagraphsRegion';
 import Utils from '../../../utils';
 import { parseValue } from '../../../utils/data';
-import { FF_DEV_2461, FF_DEV_2669, FF_DEV_2918, FF_DEV_3666, FF_LSDV_3012, isFF } from '../../../utils/feature-flags';
+import { FF_DEV_2461, FF_DEV_2669, FF_DEV_2918, FF_DEV_3666, FF_LSDV_3012, FF_LSDV_4711, isFF } from '../../../utils/feature-flags';
 import messages from '../../../utils/messages';
 import { clamp, isDefined, isValidObjectURL } from '../../../utils/utilities';
 import ObjectBase from '../Base';
 import styles from './Paragraphs.module.scss';
 
 const isFFDev2461 = isFF(FF_DEV_2461);
+const isFFLsdv4711 = isFF(FF_LSDV_4711);
 
 /**
  * The `Paragraphs` tag displays paragraphs of text on the labeling interface. Use to label dialogue transcripts for NLP and NER projects.
@@ -350,6 +351,10 @@ const OldPlayAndSync = types.model()
     let audioStopHandler = null;
     let endDuration = 0;
     let currentId = -1;
+    let currentSourceTime = 0;
+    let hasLoadedSource = false;
+    let reloadingSource = false;
+    let wasPlayingId = -1;
 
     function stop() {
       const audio = audioRef.current;
@@ -393,8 +398,41 @@ const OldPlayAndSync = types.model()
         }
       },
 
+      handleError() {
+        if (!isFFLsdv4711) return;
+
+        const audio = audioRef.current;
+
+        // if the source has succesfully loaded before, we can try to reload it
+        // as it may be an expired presigned url or temporary network issue
+        if (audio && hasLoadedSource) {
+          hasLoadedSource = false;
+          reloadingSource = true;
+          wasPlayingId = self.playingId;
+          currentSourceTime = isNaN(audio.currentTime) ? currentSourceTime : audio.currentTime;
+          stop();
+          audio.load();
+        }
+      },
+
+      handleCanPlay() {
+        if (!isFFLsdv4711) return;
+
+        const audio = audioRef.current;
+
+        hasLoadedSource = true;
+
+        if (audio && reloadingSource) {
+          reloadingSource = false;
+          audio.currentTime = currentSourceTime;
+
+          if (wasPlayingId > -1) self.play(wasPlayingId);
+        }
+      },
+
       handleSyncSeek(time) {
         if (audioRef.current) {
+          currentSourceTime = time;
           audioRef.current.currentTime = time;
         }
       },
@@ -662,6 +700,4 @@ const paragraphModelMixins = [
   ParagraphsLoadingModel,
 ].filter(Boolean);
 
-export const ParagraphsModel = types.compose('ParagraphsModel',
-  ...paragraphModelMixins,
-);
+export const ParagraphsModel = types.compose('ParagraphsModel', ...paragraphModelMixins);

src/utils/feature-flags.ts

@@ -212,6 +212,11 @@ export const FF_LSDV_4701 = 'fflag_feat_front_lsdv_4701_audio_default_decoder_ff
  */
 export const FF_LSDV_4659 = 'fflag_feat_front_lsdv_4659_skipduplicates_060323_short';
 
+/**
+ * Fixes how presigned urls are generated and accessed to remove possibility of CORS errors.
+ */
+export const FF_LSDV_4711 = 'fflag_fix_all_lsdv_4711_cors_errors_accessing_task_data_short';
+
 /**
  * Preventing creating duplicates in TextArea results with "skipDuplicates" parameter during editing.
  * It also prevent creating new history steps on every change during editing textarea results.