Early concept for rendering the frequency waveform

Author: turt2live

res/css/_components.scss

@@ -246,6 +246,7 @@
 @import "./views/toasts/_AnalyticsToast.scss";
 @import "./views/toasts/_NonUrgentEchoFailureToast.scss";
 @import "./views/verification/_VerificationShowSas.scss";
+@import "./views/voice_messages/_FrequencyBars.scss";
 @import "./views/voip/_CallContainer.scss";
 @import "./views/voip/_CallView.scss";
 @import "./views/voip/_DialPad.scss";

res/css/views/voice_messages/_FrequencyBars.scss

@@ -0,0 +1,34 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.mx_FrequencyBars {
+    position: relative;
+    height: 30px; // tallest bar can only be 30px
+
+    display: flex;
+    align-items: center; // so the bars grow from the middle
+
+    .mx_FrequencyBars_bar {
+        width: 2px;
+        margin-left: 1px;
+        margin-right: 1px;
+        background-color: $muted-fg-color;
+        display: inline-block;
+        min-height: 2px;
+        max-height: 100%;
+        border-radius: 2px; // give them soft endcaps
+    }
+}

src/components/views/rooms/VoiceRecordComposerTile.tsx

@@ -21,6 +21,7 @@ import {VoiceRecorder} from "../../../voice/VoiceRecorder";
 import {Room} from "matrix-js-sdk/src/models/room";
 import {MatrixClientPeg} from "../../../MatrixClientPeg";
 import classNames from "classnames";
+import FrequencyBars from "../voice_messages/FrequencyBars";
 
 interface IProps {
     room: Room;
@@ -57,10 +58,6 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
         const recorder = new VoiceRecorder(MatrixClientPeg.get());
         await recorder.start();
         this.props.onRecording(true);
-        // TODO: @@ TravisR: Run through EQ component
-        // recorder.frequencyData.onUpdate((freq) => {
-        //     console.log('@@ UPDATE', freq);
-        // });
         this.setState({recorder});
     };
 
@@ -71,18 +68,21 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
             'mx_VoiceRecordComposerTile_stop': !!this.state.recorder,
         });
 
+        let bars = null;
         let tooltip = _t("Record a voice message");
         if (!!this.state.recorder) {
             // TODO: @@ TravisR: Change to match behaviour
             tooltip = _t("Stop & send recording");
+            bars = <FrequencyBars recorder={this.state.recorder} />;
         }
 
-        return (
+        return (<>
+            {bars}
             <AccessibleTooltipButton
                 className={classes}
                 onClick={this.onStartStopVoiceMessage}
                 title={tooltip}
             />
-        );
+        </>);
     }
 }

src/components/views/voice_messages/FrequencyBars.tsx

@@ -0,0 +1,58 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {IFrequencyPackage, VoiceRecorder} from "../../../voice/VoiceRecorder";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import {arrayFastResample, arraySeed} from "../../../utils/arrays";
+import {percentageOf} from "../../../utils/numbers";
+
+interface IProps {
+    recorder: VoiceRecorder
+}
+
+interface IState {
+    heights: number[];
+}
+
+const DOWNSAMPLE_TARGET = 35; // number of bars
+
+@replaceableComponent("views.voice_messages.FrequencyBars")
+export default class FrequencyBars extends React.PureComponent<IProps, IState> {
+    public constructor(props) {
+        super(props);
+
+        this.state = {heights: arraySeed(0, DOWNSAMPLE_TARGET)};
+        this.props.recorder.frequencyData.onUpdate(this.onFrequencyData);
+    }
+
+    private onFrequencyData = (freq: IFrequencyPackage) => {
+        // We're downsampling from about 1024 points to about 35, so this function is fine (see docs/impl)
+        const bars = arrayFastResample(Array.from(freq.dbBars), DOWNSAMPLE_TARGET);
+        this.setState({
+            // Values are somewhat arbitrary, but help decide what shape the graph should be
+            heights: bars.map(b => percentageOf(b, -150, -70) * 100),
+        });
+    };
+
+    public render() {
+        return <div className='mx_FrequencyBars'>
+            {this.state.heights.map((h, i) => {
+                return <span key={i} style={{height: h + '%'}} className='mx_FrequencyBars_bar' />;
+            })}
+        </div>;
+    }
+}

src/utils/arrays.ts

@@ -14,6 +14,41 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+/**
+ * Quickly resample an array to have less data points. This isn't a perfect representation,
+ * though this does work best if given a large array to downsample to a much smaller array.
+ * @param {number[]} input The input array to downsample.
+ * @param {number} points The number of samples to end up with.
+ * @returns {number[]} The downsampled array.
+ */
+export function arrayFastResample(input: number[], points: number): number[] {
+    // Heavily inpired by matrix-media-repo (used with permission)
+    // https://github.com/turt2live/matrix-media-repo/blob/abe72c87d2e29/util/util_audio/fastsample.go#L10
+    const everyNth = Math.round(input.length / points);
+    const samples: number[] = [];
+    for (let i = 0; i < input.length; i += everyNth) {
+        samples.push(input[i]);
+    }
+    while (samples.length < points) {
+        samples.push(input[input.length - 1]);
+    }
+    return samples;
+}
+
+/**
+ * Creates an array of the given length, seeded with the given value.
+ * @param {T} val The value to seed the array with.
+ * @param {number} length The length of the array to create.
+ * @returns {T[]} The array.
+ */
+export function arraySeed<T>(val: T, length: number): T[] {
+    const a: T[] = [];
+    for (let i = 0; i < length; i++) {
+        a.push(val);
+    }
+    return a;
+}
+
 /**
  * Clones an array as fast as possible, retaining references of the array's values.
  * @param a The array to clone. Must be defined.

src/voice/VoiceRecorder.ts

@@ -23,7 +23,7 @@ import {SimpleObservable} from "matrix-widget-api";
 const CHANNELS = 1; // stereo isn't important
 const SAMPLE_RATE = 48000; // 48khz is what WebRTC uses. 12khz is where we lose quality.
 const BITRATE = 24000; // 24kbps is pretty high quality for our use case in opus.
-const FREQ_SAMPLE_RATE = 4; // Target rate of frequency data (samples / sec). We don't need this super often.
+const FREQ_SAMPLE_RATE = 10; // Target rate of frequency data (samples / sec). We don't need this super often.
 
 export interface IFrequencyPackage {
     dbBars: Float32Array;
@@ -60,7 +60,7 @@ export class VoiceRecorder {
             },
         });
         this.recorderContext = new AudioContext({
-            latencyHint: "interactive",
+            // latencyHint: "interactive", // we don't want a latency hint (this causes data smoothing)
             sampleRate: SAMPLE_RATE, // once again, the browser will resample for us
         });
         this.recorderSource = this.recorderContext.createMediaStreamSource(this.recorderStream);