How to use parallel offthread video?

[gajraj-gan]

I have written this component which is very slow due to the Video component I am assuming. How do I optimize this by using offthread video? Right now the video on preview looks good, but comes as black on render.

import React, { useEffect, useRef, useState } from 'react';
import {
  AbsoluteFill,
  continueRender,
  delayRender,
  Sequence,
  useCurrentFrame,
  useVideoConfig,
  Video
} from 'remotion';

type CustomOffthreadVideoProps = {
  inputVideoSrc: string;
  segmentationVideoSrc: string;
};

export const CustomOffthreadVideo: React.FC<CustomOffthreadVideoProps> = ({
  inputVideoSrc,
  segmentationVideoSrc
}) => {
  const inputVideoRef = useRef<HTMLVideoElement>(null);
  const segmentationVideoRef = useRef<HTMLVideoElement>(null);
  const canvasRef = useRef<HTMLCanvasElement>(null);
  const frame = useCurrentFrame();
  const { width, height, durationInFrames } = useVideoConfig();
  const [handle] = useState(() => delayRender('Fetching dimensions...'));
  const [video1Loaded] = useState(true);
  const [video2Loaded] = useState(true);

  useEffect(() => {
    if (video1Loaded && video2Loaded) {
      continueRender(handle);
    }
  }, [video1Loaded, video2Loaded]);

  useEffect(() => {
    const inputVideo = inputVideoRef.current;
    const segmentationVideo = segmentationVideoRef.current;
    const canvas = canvasRef.current;

    if (!video1Loaded || !video2Loaded || !inputVideo || !segmentationVideo || !canvas) {
      return;
    }

    const context = canvas.getContext('2d', {
      willReadFrequently: true
    });

    if (!context) {
      return;
    }

    const renderFrame = () => {
      if (inputVideo.readyState >= 2 && segmentationVideo.readyState >= 2) {
        context.clearRect(0, 0, width, height);

        context.drawImage(inputVideo, 0, 0, width, height);
        const imageData = context.getImageData(0, 0, width, height);
        context.clearRect(0, 0, width, height);

        context.drawImage(segmentationVideo, 0, 0, width, height);
        const segmentationData = context.getImageData(0, 0, width, height);

        for (let i = 0; i < imageData.data.length; i += 4) {
          // check if black or white pixel and then assign alpha
          const alpha = segmentationData.data[i] >= 128 ? 0 : 1;
          imageData.data[i + 3] = alpha * 255;
        }

        context.putImageData(imageData, 0, 0);
      }
    };

    renderFrame();
  }, [frame, width, height, video1Loaded, video2Loaded]);

  return (
    <Sequence from={0} durationInFrames={durationInFrames} premountFor={500}>
      <AbsoluteFill>
        <Video
          ref={inputVideoRef}
          src={inputVideoSrc}
          style={{ opacity: 0 }}
          crossOrigin="anonymous"
        />
        <Video
          ref={segmentationVideoRef}
          src={segmentationVideoSrc}
          style={{ opacity: 0 }}
          crossOrigin="anonymous"
        />
      </AbsoluteFill>
      <AbsoluteFill>
        <canvas ref={canvasRef} width={width} height={height} />
      </AbsoluteFill>
    </Sequence>
  );
};