Merge pull request scratchfoundation#5376 from LLK/revert-5338-fix-long-sound

Revert "Implement sound downsampler to try to allow more sounds to be edited"

Author: paulkaplan

src/containers/sound-editor.jsx

@@ -6,12 +6,7 @@ import VM from 'scratch-vm';
 
 import {connect} from 'react-redux';
 
-import {
-    computeChunkedRMS,
-    encodeAndAddSoundToVM,
-    downsampleIfNeeded,
-    dropEveryOtherSample
-} from '../lib/audio/audio-util.js';
+import {computeChunkedRMS, encodeAndAddSoundToVM, SOUND_BYTE_LIMIT} from '../lib/audio/audio-util.js';
 import AudioEffects from '../lib/audio/audio-effects.js';
 import SoundEditorComponent from '../components/sound-editor/sound-editor.jsx';
 import AudioBufferPlayer from '../lib/audio/audio-buffer-player.js';
@@ -44,8 +39,7 @@ class SoundEditor extends React.Component {
             'paste',
             'handleKeyPress',
             'handleContainerClick',
-            'setRef',
-            'resampleBufferToRate'
+            'setRef'
         ]);
         this.state = {
             copyBuffer: null,
@@ -138,32 +132,45 @@ class SoundEditor extends React.Component {
         });
     }
     submitNewSamples (samples, sampleRate, skipUndo) {
-        return downsampleIfNeeded({samples, sampleRate}, this.resampleBufferToRate)
-            .then(({samples: newSamples, sampleRate: newSampleRate}) =>
-                WavEncoder.encode({
-                    sampleRate: newSampleRate,
-                    channelData: [newSamples]
-                }).then(wavBuffer => {
-                    if (!skipUndo) {
-                        this.redoStack = [];
-                        if (this.undoStack.length >= UNDO_STACK_SIZE) {
-                            this.undoStack.shift(); // Drop the first element off the array
-                        }
-                        this.undoStack.push(this.getUndoItem());
-                    }
-                    this.resetState(newSamples, newSampleRate);
-                    this.props.vm.updateSoundBuffer(
-                        this.props.soundIndex,
-                        this.audioBufferPlayer.buffer,
-                        new Uint8Array(wavBuffer));
-                    return true; // Edit was successful
-                })
-            )
-            .catch(e => {
-                // Encoding failed, or the sound was too large to save so edit is rejected
-                log.error(`Encountered error while trying to encode sound update: ${e}`);
-                return false; // Edit was not applied
+        // Encode the new sound into a wav so that it can be stored
+        let wavBuffer = null;
+        try {
+            wavBuffer = WavEncoder.encode.sync({
+                sampleRate: sampleRate,
+                channelData: [samples]
             });
+
+            if (wavBuffer.byteLength > SOUND_BYTE_LIMIT) {
+                // Cancel the sound update by setting to null
+                wavBuffer = null;
+                log.error(`Refusing to encode sound larger than ${SOUND_BYTE_LIMIT} bytes`);
+            }
+        } catch (e) {
+            // This error state is mostly for the mock sounds used during testing.
+            // Any incorrect sound buffer trying to get interpretd as a Wav file
+            // should yield this error.
+            // This can also happen if the sound is too be allocated in memory.
+            log.error(`Encountered error while trying to encode sound update: ${e}`);
+        }
+
+        // Do not submit sound if it could not be encoded (i.e. if too large)
+        if (wavBuffer) {
+            if (!skipUndo) {
+                this.redoStack = [];
+                if (this.undoStack.length >= UNDO_STACK_SIZE) {
+                    this.undoStack.shift(); // Drop the first element off the array
+                }
+                this.undoStack.push(this.getUndoItem());
+            }
+            this.resetState(samples, sampleRate);
+            this.props.vm.updateSoundBuffer(
+                this.props.soundIndex,
+                this.audioBufferPlayer.buffer,
+                new Uint8Array(wavBuffer));
+
+            return true; // Update succeeded
+        }
+        return false; // Update failed
     }
     handlePlay () {
         this.audioBufferPlayer.stop();
@@ -202,15 +209,13 @@ class SoundEditor extends React.Component {
             newSamples.set(firstPart, 0);
             newSamples.set(secondPart, firstPart.length);
         }
-        this.submitNewSamples(newSamples, sampleRate).then(() => {
-            this.setState({
-                trimStart: null,
-                trimEnd: null
-            });
+        this.submitNewSamples(newSamples, sampleRate);
+        this.setState({
+            trimStart: null,
+            trimEnd: null
         });
     }
     handleDeleteInverse () {
-        // Delete everything outside of the trimmers
         const {samples, sampleRate} = this.copyCurrentBuffer();
         const sampleCount = samples.length;
         const startIndex = Math.floor(this.state.trimStart * sampleCount);
@@ -219,13 +224,10 @@ class SoundEditor extends React.Component {
         if (clippedSamples.length === 0) {
             clippedSamples = new Float32Array(1);
         }
-        this.submitNewSamples(clippedSamples, sampleRate).then(success => {
-            if (success) {
-                this.setState({
-                    trimStart: null,
-                    trimEnd: null
-                });
-            }
+        this.submitNewSamples(clippedSamples, sampleRate);
+        this.setState({
+            trimStart: null,
+            trimEnd: null
         });
     }
     handleUpdateTrim (trimStart, trimEnd) {
@@ -255,15 +257,14 @@ class SoundEditor extends React.Component {
         effects.process((renderedBuffer, adjustedTrimStart, adjustedTrimEnd) => {
             const samples = renderedBuffer.getChannelData(0);
             const sampleRate = renderedBuffer.sampleRate;
-            this.submitNewSamples(samples, sampleRate).then(success => {
-                if (success) {
-                    if (this.state.trimStart === null) {
-                        this.handlePlay();
-                    } else {
-                        this.setState({trimStart: adjustedTrimStart, trimEnd: adjustedTrimEnd}, this.handlePlay);
-                    }
+            const success = this.submitNewSamples(samples, sampleRate);
+            if (success) {
+                if (this.state.trimStart === null) {
+                    this.handlePlay();
+                } else {
+                    this.setState({trimStart: adjustedTrimStart, trimEnd: adjustedTrimEnd}, this.handlePlay);
                 }
-            });
+            }
         });
     }
     tooLoud () {
@@ -286,22 +287,16 @@ class SoundEditor extends React.Component {
         this.redoStack.push(this.getUndoItem());
         const {samples, sampleRate, trimStart, trimEnd} = this.undoStack.pop();
         if (samples) {
-            return this.submitNewSamples(samples, sampleRate, true).then(success => {
-                if (success) {
-                    this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
-                }
-            });
+            this.submitNewSamples(samples, sampleRate, true);
+            this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
         }
     }
     handleRedo () {
         const {samples, sampleRate, trimStart, trimEnd} = this.redoStack.pop();
         if (samples) {
             this.undoStack.push(this.getUndoItem());
-            return this.submitNewSamples(samples, sampleRate, true).then(success => {
-                if (success) {
-                    this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
-                }
-            });
+            this.submitNewSamples(samples, sampleRate, true);
+            this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
         }
     }
     handleCopy () {
@@ -327,39 +322,25 @@ class SoundEditor extends React.Component {
         });
     }
     resampleBufferToRate (buffer, newRate) {
-        return new Promise((resolve, reject) => {
-            const sampleRateRatio = newRate / buffer.sampleRate;
-            const newLength = sampleRateRatio * buffer.samples.length;
-            let offlineContext;
-            // Try to use either OfflineAudioContext or webkitOfflineAudioContext to resample
-            // The constructors will throw if trying to resample at an unsupported rate
-            // (e.g. Safari/webkitOAC does not support lower than 44khz).
-            try {
-                if (window.OfflineAudioContext) {
-                    offlineContext = new window.OfflineAudioContext(1, newLength, newRate);
-                } else if (window.webkitOfflineAudioContext) {
-                    offlineContext = new window.webkitOfflineAudioContext(1, newLength, newRate);
-                }
-            } catch {
-                // If no OAC available and downsampling by 2, downsample by dropping every other sample.
-                if (newRate === buffer.sampleRate / 2) {
-                    return resolve(dropEveryOtherSample(buffer));
-                }
-                return reject('Could not resample');
+        return new Promise(resolve => {
+            if (window.OfflineAudioContext) {
+                const sampleRateRatio = newRate / buffer.sampleRate;
+                const newLength = sampleRateRatio * buffer.samples.length;
+                const offlineContext = new window.OfflineAudioContext(1, newLength, newRate);
+                const source = offlineContext.createBufferSource();
+                const audioBuffer = offlineContext.createBuffer(1, buffer.samples.length, buffer.sampleRate);
+                audioBuffer.getChannelData(0).set(buffer.samples);
+                source.buffer = audioBuffer;
+                source.connect(offlineContext.destination);
+                source.start();
+                offlineContext.startRendering();
+                offlineContext.oncomplete = ({renderedBuffer}) => {
+                    resolve({
+                        samples: renderedBuffer.getChannelData(0),
+                        sampleRate: newRate
+                    });
+                };
             }
-            const source = offlineContext.createBufferSource();
-            const audioBuffer = offlineContext.createBuffer(1, buffer.samples.length, buffer.sampleRate);
-            audioBuffer.getChannelData(0).set(buffer.samples);
-            source.buffer = audioBuffer;
-            source.connect(offlineContext.destination);
-            source.start();
-            offlineContext.startRendering();
-            offlineContext.oncomplete = ({renderedBuffer}) => {
-                resolve({
-                    samples: renderedBuffer.getChannelData(0),
-                    sampleRate: newRate
-                });
-            };
         });
     }
     paste () {
@@ -370,11 +351,8 @@ class SoundEditor extends React.Component {
             const newSamples = new Float32Array(newLength);
             newSamples.set(samples, 0);
             newSamples.set(this.state.copyBuffer.samples, samples.length);
-            this.submitNewSamples(newSamples, this.props.sampleRate, false).then(success => {
-                if (success) {
-                    this.handlePlay();
-                }
-            });
+            this.submitNewSamples(newSamples, this.props.sampleRate, false);
+            this.handlePlay();
         } else {
             // else replace the selection with the pasted sound
             const trimStartSamples = this.state.trimStart * samples.length;
@@ -393,14 +371,11 @@ class SoundEditor extends React.Component {
             const newDurationSeconds = newSamples.length / this.state.copyBuffer.sampleRate;
             const adjustedTrimStart = trimStartSeconds / newDurationSeconds;
             const adjustedTrimEnd = trimEndSeconds / newDurationSeconds;
-            this.submitNewSamples(newSamples, this.props.sampleRate, false).then(success => {
-                if (success) {
-                    this.setState({
-                        trimStart: adjustedTrimStart,
-                        trimEnd: adjustedTrimEnd
-                    }, this.handlePlay);
-                }
-            });
+            this.submitNewSamples(newSamples, this.props.sampleRate, false);
+            this.setState({
+                trimStart: adjustedTrimStart,
+                trimEnd: adjustedTrimEnd
+            }, this.handlePlay);
         }
     }
     handlePaste () {

src/lib/audio/audio-util.js

@@ -59,57 +59,9 @@ const encodeAndAddSoundToVM = function (vm, samples, sampleRate, name, callback)
     });
 };
 
-/**
- @typedef SoundBuffer
- @type {Object}
- @property {Float32Array} samples Array of audio samples
- @property {number} sampleRate Audio sample rate
- */
-
-/**
- * Downsample the given buffer to try to reduce file size below SOUND_BYTE_LIMIT
- * @param {SoundBuffer} buffer - Buffer to resample
- * @param {function(SoundBuffer):Promise<SoundBuffer>} resampler - resampler function
- * @returns {SoundBuffer} Downsampled buffer with half the sample rate
- */
-const downsampleIfNeeded = (buffer, resampler) => {
-    const {samples, sampleRate} = buffer;
-    const duration = samples.length / sampleRate;
-    const encodedByteLength = samples.length * 2; /* bitDepth 16 bit */
-    // Resolve immediately if already within byte limit
-    if (encodedByteLength < SOUND_BYTE_LIMIT) {
-        return Promise.resolve({samples, sampleRate});
-    }
-    // If encodeable at 22khz, resample and call submitNewSamples again
-    if (duration * 22050 * 2 < SOUND_BYTE_LIMIT) {
-        return resampler({samples, sampleRate}, 22050);
-    }
-    // Cannot save this sound at 22khz, refuse to edit
-    // In the future we could introduce further compression here
-    return Promise.reject('Sound too large to save, refusing to edit');
-};
-
-/**
- * Drop every other sample of an audio buffer as a last-resort way of downsampling.
- * @param {SoundBuffer} buffer - Buffer to resample
- * @returns {SoundBuffer} Downsampled buffer with half the sample rate
- */
-const dropEveryOtherSample = buffer => {
-    const newLength = Math.floor(buffer.samples.length / 2);
-    const newSamples = new Float32Array(newLength);
-    for (let i = 0; i < newLength; i++) {
-        newSamples[i] = buffer.samples[i * 2];
-    }
-    return {
-        samples: newSamples,
-        sampleRate: buffer.rate / 2
-    };
-};
-
 export {
     computeRMS,
     computeChunkedRMS,
     encodeAndAddSoundToVM,
-    downsampleIfNeeded,
-    dropEveryOtherSample
+    SOUND_BYTE_LIMIT
 };

test/__mocks__/audio-effects.js

@@ -14,10 +14,7 @@ export default class MockAudioEffects {
         this.buffer = buffer;
         this.name = name;
         this.process = jest.fn(done => {
-            this._finishProcessing = renderedBuffer => {
-                done(renderedBuffer, 0, 1);
-                return new Promise(resolve => setTimeout(resolve));
-            };
+            this._finishProcessing = renderedBuffer => done(renderedBuffer, 0, 1);
         });
         MockAudioEffects.instance = this;
     }