Merge pull request #5801 from matrix-org/travis/voice-messages/waveform

Show waveform and timer in voice messages

Author: turt2live

res/css/_components.scss

@@ -247,6 +247,7 @@
 @import "./views/toasts/_AnalyticsToast.scss";
 @import "./views/toasts/_NonUrgentEchoFailureToast.scss";
 @import "./views/verification/_VerificationShowSas.scss";
+@import "./views/voice_messages/_Waveform.scss";
 @import "./views/voip/_CallContainer.scss";
 @import "./views/voip/_CallView.scss";
 @import "./views/voip/_DialPad.scss";

res/css/views/rooms/_VoiceRecordComposerTile.scss

@@ -34,3 +34,43 @@ limitations under the License.
         background-color: $voice-record-stop-symbol-color;
     }
 }
+
+.mx_VoiceRecordComposerTile_waveformContainer {
+    padding: 5px;
+    padding-right: 4px; // there's 1px from the waveform itself, so account for that
+    padding-left: 15px; // +10px for the live circle, +5px for regular padding
+    background-color: $voice-record-waveform-bg-color;
+    border-radius: 12px;
+    margin-right: 12px; // isolate from stop button
+
+    // Cheat at alignment a bit
+    display: flex;
+    align-items: center;
+
+    position: relative; // important for the live circle
+
+    color: $voice-record-waveform-fg-color;
+    font-size: $font-14px;
+
+    &::before {
+        // TODO: @@ TravisR: Animate
+        content: '';
+        background-color: $voice-record-live-circle-color;
+        width: 10px;
+        height: 10px;
+        position: absolute;
+        left: 8px;
+        top: 16px; // vertically center
+        border-radius: 10px;
+    }
+
+    .mx_Waveform_bar {
+        background-color: $voice-record-waveform-fg-color;
+    }
+
+    .mx_Clock {
+        padding-right: 8px; // isolate from waveform
+        padding-left: 10px; // isolate from live circle
+        width: 42px; // we're not using a monospace font, so fake it
+    }
+}

res/css/views/voice_messages/_Waveform.scss

@@ -0,0 +1,40 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.mx_Waveform {
+    position: relative;
+    height: 30px; // tallest bar can only be 30px
+    top: 1px; // because of our border trick (see below), we're off by 1px of aligntment
+
+    display: flex;
+    align-items: center; // so the bars grow from the middle
+
+    overflow: hidden; // this is cheaper than a `max-height: calc(100% - 4px)` in the bar's CSS.
+
+    // A bar is meant to be a 2x2 circle when at zero height, and otherwise a 2px wide line
+    // with rounded caps.
+    .mx_Waveform_bar {
+        width: 0; // 0px width means we'll end up using the border as our width
+        border: 1px solid transparent; // transparent means we'll use the background colour
+        border-radius: 2px; // rounded end caps, based on the border
+        min-height: 0; // like the width, we'll rely on the border to give us height
+        max-height: 100%; // this makes the `height: 42%` work on the element
+        margin-left: 1px; // we want 2px between each bar, so 1px on either side for balance
+        margin-right: 1px;
+
+        // background color is handled by the parent components
+    }
+}

res/themes/legacy-light/css/_legacy-light.scss

@@ -190,6 +190,9 @@ $groupFilterPanel-divider-color: $roomlist-header-color;
 
 $voice-record-stop-border-color: #E3E8F0;
 $voice-record-stop-symbol-color: $warning-color;
+$voice-record-waveform-bg-color: #E3E8F0;
+$voice-record-waveform-fg-color: $muted-fg-color;
+$voice-record-live-circle-color: $warning-color;
 
 $roomtile-preview-color: #9e9e9e;
 $roomtile-default-badge-bg-color: #61708b;

res/themes/light/css/_light.scss

@@ -181,6 +181,9 @@ $groupFilterPanel-divider-color: $roomlist-header-color;
 
 $voice-record-stop-border-color: #E3E8F0;
 $voice-record-stop-symbol-color: $warning-color;
+$voice-record-waveform-bg-color: #E3E8F0;
+$voice-record-waveform-fg-color: $muted-fg-color;
+$voice-record-live-circle-color: $warning-color;
 
 $roomtile-preview-color: $secondary-fg-color;
 $roomtile-default-badge-bg-color: #61708b;

src/components/views/rooms/VoiceRecordComposerTile.tsx

@@ -21,6 +21,9 @@ import {VoiceRecorder} from "../../../voice/VoiceRecorder";
 import {Room} from "matrix-js-sdk/src/models/room";
 import {MatrixClientPeg} from "../../../MatrixClientPeg";
 import classNames from "classnames";
+import LiveRecordingWaveform from "../voice_messages/LiveRecordingWaveform";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import LiveRecordingClock from "../voice_messages/LiveRecordingClock";
 
 interface IProps {
     room: Room;
@@ -31,6 +34,10 @@ interface IState {
     recorder?: VoiceRecorder;
 }
 
+/**
+ * Container tile for rendering the voice message recorder in the composer.
+ */
+@replaceableComponent("views.rooms.VoiceRecordComposerTile")
 export default class VoiceRecordComposerTile extends React.PureComponent<IProps, IState> {
     public constructor(props) {
         super(props);
@@ -57,13 +64,18 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
         const recorder = new VoiceRecorder(MatrixClientPeg.get());
         await recorder.start();
         this.props.onRecording(true);
-        // TODO: @@ TravisR: Run through EQ component
-        // recorder.frequencyData.onUpdate((freq) => {
-        //     console.log('@@ UPDATE', freq);
-        // });
         this.setState({recorder});
     };
 
+    private renderWaveformArea() {
+        if (!this.state.recorder) return null;
+
+        return <div className='mx_VoiceRecordComposerTile_waveformContainer'>
+            <LiveRecordingClock recorder={this.state.recorder} />
+            <LiveRecordingWaveform recorder={this.state.recorder} />
+        </div>;
+    }
+
     public render() {
         const classes = classNames({
             'mx_MessageComposer_button': !this.state.recorder,
@@ -77,12 +89,13 @@ export default class VoiceRecordComposerTile extends React.PureComponent<IProps,
             tooltip = _t("Stop & send recording");
         }
 
-        return (
+        return (<>
+            {this.renderWaveformArea()}
             <AccessibleTooltipButton
                 className={classes}
                 onClick={this.onStartStopVoiceMessage}
                 title={tooltip}
             />
-        );
+        </>);
     }
 }

src/components/views/voice_messages/Clock.tsx

@@ -0,0 +1,42 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+
+interface IProps {
+    seconds: number;
+}
+
+interface IState {
+}
+
+/**
+ * Simply converts seconds into minutes and seconds. Note that hours will not be
+ * displayed, making it possible to see "82:29".
+ */
+@replaceableComponent("views.voice_messages.Clock")
+export default class Clock extends React.PureComponent<IProps, IState> {
+    public constructor(props) {
+        super(props);
+    }
+
+    public render() {
+        const minutes = Math.floor(this.props.seconds / 60).toFixed(0).padStart(2, '0');
+        const seconds = Math.round(this.props.seconds % 60).toFixed(0).padStart(2, '0'); // hide millis
+        return <span className='mx_Clock'>{minutes}:{seconds}</span>;
+    }
+}

src/components/views/voice_messages/LiveRecordingClock.tsx

@@ -0,0 +1,55 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {IRecordingUpdate, VoiceRecorder} from "../../../voice/VoiceRecorder";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import Clock from "./Clock";
+
+interface IProps {
+    recorder: VoiceRecorder;
+}
+
+interface IState {
+    seconds: number;
+}
+
+/**
+ * A clock for a live recording.
+ */
+@replaceableComponent("views.voice_messages.LiveRecordingClock")
+export default class LiveRecordingClock extends React.Component<IProps, IState> {
+    public constructor(props) {
+        super(props);
+
+        this.state = {seconds: 0};
+        this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
+    }
+
+    shouldComponentUpdate(nextProps: Readonly<IProps>, nextState: Readonly<IState>, nextContext: any): boolean {
+        const currentFloor = Math.floor(this.state.seconds);
+        const nextFloor = Math.floor(nextState.seconds);
+        return currentFloor !== nextFloor;
+    }
+
+    private onRecordingUpdate = (update: IRecordingUpdate) => {
+        this.setState({seconds: update.timeSeconds});
+    };
+
+    public render() {
+        return <Clock seconds={this.state.seconds} />;
+    }
+}

src/components/views/voice_messages/LiveRecordingWaveform.tsx

@@ -0,0 +1,62 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {IRecordingUpdate, VoiceRecorder} from "../../../voice/VoiceRecorder";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+import {arrayFastResample, arraySeed} from "../../../utils/arrays";
+import {percentageOf} from "../../../utils/numbers";
+import Waveform from "./Waveform";
+
+interface IProps {
+    recorder: VoiceRecorder;
+}
+
+interface IState {
+    heights: number[];
+}
+
+const DOWNSAMPLE_TARGET = 35; // number of bars we want
+
+/**
+ * A waveform which shows the waveform of a live recording
+ */
+@replaceableComponent("views.voice_messages.LiveRecordingWaveform")
+export default class LiveRecordingWaveform extends React.PureComponent<IProps, IState> {
+    public constructor(props) {
+        super(props);
+
+        this.state = {heights: arraySeed(0, DOWNSAMPLE_TARGET)};
+        this.props.recorder.liveData.onUpdate(this.onRecordingUpdate);
+    }
+
+    private onRecordingUpdate = (update: IRecordingUpdate) => {
+        // The waveform and the downsample target are pretty close, so we should be fine to
+        // do this, despite the docs on arrayFastResample.
+        const bars = arrayFastResample(Array.from(update.waveform), DOWNSAMPLE_TARGET);
+        this.setState({
+            // The incoming data is between zero and one, but typically even screaming into a
+            // microphone won't send you over 0.6, so we artificially adjust the gain for the
+            // waveform. This results in a slightly more cinematic/animated waveform for the
+            // user.
+            heights: bars.map(b => percentageOf(b, 0, 0.50)),
+        });
+    };
+
+    public render() {
+        return <Waveform relHeights={this.state.heights} />;
+    }
+}

src/components/views/voice_messages/Waveform.tsx

@@ -0,0 +1,45 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+import {replaceableComponent} from "../../../utils/replaceableComponent";
+
+interface IProps {
+    relHeights: number[]; // relative heights (0-1)
+}
+
+interface IState {
+}
+
+/**
+ * A simple waveform component. This renders bars (centered vertically) for each
+ * height provided in the component properties. Updating the properties will update
+ * the rendered waveform.
+ */
+@replaceableComponent("views.voice_messages.Waveform")
+export default class Waveform extends React.PureComponent<IProps, IState> {
+    public constructor(props) {
+        super(props);
+    }
+
+    public render() {
+        return <div className='mx_Waveform'>
+            {this.props.relHeights.map((h, i) => {
+                return <span key={i} style={{height: (h * 100) + '%'}} className='mx_Waveform_bar' />;
+            })}
+        </div>;
+    }
+}