Record Desktop Audio (#1079)

This PR allows Studio to record desktop audio and resolves #551

Unfortunately, not all browsers are supported yet, as you can see
[here](https://caniuse.com/mdn-api_mediadevices_getdisplaymedia_audio_capture_support)
and as Lukas mentioned here:
[#551](#551)

The users get a hint after selecting display as source, with which
browsers they can record desktop audio.


Output of the recordings for download:

**Only camera recorded:** Nothing changed, the finished camera video
will contain microphone audio (if recorded)

**Only display recorded:** If the user recorded both, desktop audio and
microphone audio, the finished video will have both audio tracks.

**Display and camera recorded:** The finished desktop video will contain
desktop audio and the finished camera video will contain the microphone
audio

**One thing/problem:**
In the "Review & trim" step (also after uploading to Opencast), if both
the display AND the camera were recorded: you only hear the microphone
audio or nothing (depending on whether microphone audio was recorded or
not). I'm not entirely sure why.
The audio is correct and audible for each of the videos that can be
downloaded in the last step in Studio.

Author: LukasKalbertodt

src/capturer.tsx

@@ -55,7 +55,7 @@ export async function startDisplayCapture(
       ...videoConstraints,
       ...height,
     },
-    audio: false,
+    audio: true,
   };
 
   try {

src/i18n/locales/de.json

@@ -55,7 +55,9 @@
       "aspect-ratio-auto": "auto",
       "quality": "Qualität",
       "quality-auto": "auto",
-      "preferences-note": "<0>Hinweis:</0> Dies sind lediglich Präferenzen. Es ist nicht garantiert, dass alle Einstellungen von Ihrem Gerät unterstützt werden. Im Zweifelsfall 'auto' wählen."
+      "preferences-note": "<0>Hinweis:</0> Dies sind lediglich Präferenzen. Es ist nicht garantiert, dass alle Einstellungen von Ihrem Gerät unterstützt werden. Im Zweifelsfall 'auto' wählen.",
+      "display-audio-shared": "Bildschirmton wird aufgezeichnet.",
+      "display-audio-not-shared": "Bildschirmton wird nicht aufgezeichnet. <0>Hinweis</0>: Nicht alle Browser und Betriebssysteme unterstützen die Aufnahme des Bildschirmtons."
     },
     "audio": {
       "label": "Audioquelle auswählen",

src/i18n/locales/en.json

@@ -55,7 +55,9 @@
       "aspect-ratio-auto": "auto",
       "quality": "Quality",
       "quality-auto": "auto",
-      "preferences-note": "<0>Note:</0> these are merely preferences and it cannot be guaranteed that all options are actually supported on your device. If in doubt, choose 'auto'."
+      "preferences-note": "<0>Note:</0> these are merely preferences and it cannot be guaranteed that all options are actually supported on your device. If in doubt, choose 'auto'.",
+      "display-audio-shared": "Display audio will be recorded.",
+      "display-audio-not-shared": "Display audio will not be recorded. <0>Note</0>: Not all browsers and operating systems support display audio capture."
     },
     "audio": {
       "label": "Select audio source",

src/steps/recording/index.tsx

@@ -36,12 +36,29 @@ const addRecordOnStop = (
   };
 };
 
-const mixAudioIntoVideo = (audioStream: MediaStream | null, videoStream: MediaStream) => {
-  if (!(audioStream?.getAudioTracks().length)) {
-    return videoStream;
-  }
-  return new MediaStream([...videoStream.getVideoTracks(), ...audioStream.getAudioTracks()]);
-};
+const mixAudioIntoVideo = (audioStreams: (MediaStream | null)[], videoStream: MediaStream) => (
+  audioStreams.reduce<MediaStream>(
+    (stream, audioStream) => audioStream?.getAudioTracks().length
+      ? new MediaStream([
+        ...stream.getVideoTracks(),
+        ...(
+          stream.getAudioTracks().length
+            ? (() => {
+              const audioContext = new AudioContext();
+              const accumulatedAudio = audioContext.createMediaStreamSource(stream);
+              const currentAudio = audioContext.createMediaStreamSource(audioStream);
+              const resultAudio = audioContext.createMediaStreamDestination();
+              accumulatedAudio.connect(resultAudio);
+              currentAudio.connect(resultAudio);
+              return resultAudio.stream;
+            })()
+            : audioStream
+        ).getAudioTracks(),
+      ])
+      : stream,
+    videoStream,
+  )
+);
 
 
 export const Recording: React.FC<StepProps> = ({ goToNextStep, goToPrevStep }) => {
@@ -70,14 +87,13 @@ export const Recording: React.FC<StepProps> = ({ goToNextStep, goToPrevStep }) =
 
     if (displayStream) {
       const onStop = addRecordOnStop(dispatch, "desktop");
-      const stream = mixAudioIntoVideo(state.audioStream, displayStream);
+      const stream = mixAudioIntoVideo([state.audioStream], displayStream);
       desktopRecorder.current = new Recorder(stream, settings.recording, onStop);
       desktopRecorder.current.start();
     }
-
     if (userStream) {
       const onStop = addRecordOnStop(dispatch, "video");
-      const stream = mixAudioIntoVideo(state.audioStream, userStream);
+      const stream = mixAudioIntoVideo([state.audioStream, displayStream], userStream);
       videoRecorder.current = new Recorder(stream, settings.recording, onStop);
       videoRecorder.current.start();
     }

src/steps/video-setup/prefs.tsx

@@ -18,6 +18,7 @@ import {
   stopUserCapture,
 } from "../../capturer";
 import { Select } from "../../ui/Select";
+import { OVERLAY_STYLE } from "./preview";
 
 
 /**
@@ -218,29 +219,14 @@ export const StreamSettings: React.FC<StreamSettingsProps> = ({ isDesktop, strea
             onClick={() => setIsExpanded(old => !old)}
             aria-label={label}
             css={{
-              border: "none",
-              display: "inline-block",
-              backgroundColor: "rgba(0, 0, 0, 0.3)",
-              color: "white",
-              padding: 8,
+              ...OVERLAY_STYLE,
               fontSize: 26,
-              backdropFilter: "invert(0.3) blur(4px)",
-              lineHeight: 0,
-              borderRadius: "10px",
-              cursor: "pointer",
-              "&:hover, &:focus-visible": {
-                backgroundColor: "rgba(0, 0, 0, 0.5)",
-              },
               "> svg": {
                 transition: "transform 0.2s",
               },
               "&:hover > svg, &:focus > svg": {
                 transform: isExpanded ? "none" : "rotate(45deg)",
               },
-              "&:focus-visible": {
-                outline: "5px dashed white",
-                outlineOffset: -2.5,
-              },
             }}
           >
             {isExpanded ? <FiX /> : <FiSettings />}

src/steps/video-setup/preview.tsx

@@ -1,12 +1,13 @@
 import { useEffect, useRef } from "react";
-import { Spinner, match, unreachable, useColorScheme } from "@opencast/appkit";
-import { useTranslation } from "react-i18next";
+import { Spinner, WithTooltip, match, unreachable, useColorScheme } from "@opencast/appkit";
+import { Trans, useTranslation } from "react-i18next";
+import { LuInfo, LuVolume2, LuVolumeX } from "react-icons/lu";
 
 import { COLORS, dimensionsOf } from "../../util";
-import { StreamSettings } from "./prefs";
-import { Input } from ".";
 import { VideoBox, useVideoBoxResize } from "../../ui/VideoBox";
 import { ErrorBox } from "../../ui/ErrorBox";
+import { StreamSettings } from "./prefs";
+import { Input } from ".";
 
 
 
@@ -63,7 +64,10 @@ const StreamPreview: React.FC<{ input: Input }> = ({ input }) => {
       },
     }}>
       <PreviewVideo input={input} />
-      {input.stream && <StreamSettings isDesktop={input.isDesktop} stream={input.stream} />}
+      {input.stream && <>
+        {input.isDesktop && <DisplayAudioInfo stream={input.stream} />}
+        <StreamSettings isDesktop={input.isDesktop} stream={input.stream} />
+      </>}
     </div>
   );
 };
@@ -145,3 +149,49 @@ const PreviewVideo: React.FC<{ input: Input }> = ({ input }) => {
     </div>
   );
 };
+
+export const DisplayAudioInfo: React.FC<{ stream: MediaStream }> = ({ stream }) => {
+  const hasAudio = stream.getAudioTracks().length;
+
+  return (
+    <div css={{
+      position: "absolute",
+      top: 8,
+      right: 8,
+    }}>
+      <WithTooltip
+        placement="top"
+        tooltip={
+          <Trans i18nKey={
+            `steps.video.${hasAudio ? "display-audio-shared" : "display-audio-not-shared"}`
+          }>
+            <strong>Note:</strong> Explanation.
+          </Trans>
+        }
+      >
+        <div css={{ ...OVERLAY_STYLE, fontSize: 15 }}>
+          <LuInfo /> {hasAudio ? <LuVolume2 /> : <LuVolumeX />}
+        </div>
+      </WithTooltip>
+    </div>
+  );
+};
+
+export const OVERLAY_STYLE = {
+  border: "none",
+  display: "inline-block",
+  backgroundColor: "rgba(0, 0, 0, 0.3)",
+  color: "white",
+  padding: 8,
+  backdropFilter: "invert(0.3) blur(4px)",
+  lineHeight: 0,
+  borderRadius: 10,
+  cursor: "pointer",
+  "&:hover, &:focus-visible": {
+    backgroundColor: "rgba(0, 0, 0, 0.5)",
+  },
+  "&:focus-visible": {
+    outline: "5px dashed white",
+    outlineOffset: -2.5,
+  },
+};