costum audio-audio plugin

[ZifengLIU143]

Hi all,

I am trying to merge the "html-audio-response" plugin and the "audio-button-response" plugin to create a plugin which allows audio file as stimulus and records audio response. Also, I want to initiate the recording as soon as the trial begins. This is the first time I try to modify plugin codes and unsurprisingly I get errors. The audio files play as I expect, but the recording does not seem to work. The script is not saving the recording to the data.

I share my codes below:

`var jsPsychAudioAudioResponse = (function (jspsych) {
'use strict';

const info = {
name: "audio-audio-response",
parameters: {
/** The audio file to be displayed /
audio_stimulus: {
type: jspsych.ParameterType.AUDIO,
pretty_name: "Audio_Stimulus",
default: undefined,
},
/* Any content here will be displayed on the screen. /
prompt: {
type: jspsych.ParameterType.HTML_STRING,
pretty_name: "Prompt",
default: null,
},
/* How long to show the trial. /
recording_duration: {
type: jspsych.ParameterType.INT,
default: 2000,
},
/* Whether or not to show a button to end the recording. If false, the recording_duration must be set. /
show_done_button: {
type: jspsych.ParameterType.BOOL,
default: true,
},
/* Label for the done (stop recording) button. Only used if show_done_button is true. /
done_button_label: {
type: jspsych.ParameterType.STRING,
default: "Continue",
},
/* Label for the record again button (only used if allow_playback is true). /
record_again_button_label: {
type: jspsych.ParameterType.STRING,
default: "Record again",
},
/* Label for the button to accept the audio recording (only used if allow_playback is true). /
accept_button_label: {
type: jspsych.ParameterType.STRING,
default: "Continue",
},
/* Whether or not to allow the participant to playback the recording and either accept or re-record. /
allow_playback: {
type: jspsych.ParameterType.BOOL,
default: false,
},
/* Whether or not to save the video URL to the trial data. /
save_audio_url: {
type: jspsych.ParameterType.BOOL,
default: false,
},
},
};
/*

• html-audio-response
• jsPsych plugin for displaying a stimulus and recording an audio response through a microphone
• @author Josh de Leeuw
• @see {@link https://www.jspsych.org/plugins/jspsych-html-audio-response/ html-audio-response plugin documentation on jspsych.org}
  */

class AudioAudioResponsePlugin {
constructor(jsPsych) {
this.jsPsych = jsPsych;
this.rt = null;
this.recorded_data_chunks = [];
// this.on_load = this.on_load.bind(this);

  }
  
  trial(display_element, trial) {
    this.recorder = this.jsPsych.pluginAPI.getMicrophoneRecorder();
    this.setupRecordingEvents(display_element, trial);
    this.loadAudio(display_element, trial);
    this.startRecording();
    // hold the .resolve() function from the Promise that ends the trial
    let trial_complete;
    if (this.recorder && this.recorder.state !== 'recording') {
        this.recorder.start();
    }
  }
  loadAudio(display_element, trial, on_load) {
    // hold the .resolve() function from the Promise that ends the trial
    let trial_complete;
    // setup stimulus
    var context = this.jsPsych.pluginAPI.audioContext();
    // store response
    // var response = {
    //     rt: null,
    //     button: null,
    // };
    // record webaudio context start time
    var startTime;
    // load audio file
    this.jsPsych.pluginAPI
        .getAudioBuffer(trial.audio_stimulus)
        .then((buffer) => {
        if (context !== null) {
            this.audio = context.createBufferSource();
            this.audio.buffer = buffer;
            this.audio.connect(context.destination);
        }
        else {
            this.audio = buffer;
            this.audio.currentTime = 0;
        }
        setupAudioTrial();
    })
        .catch((err) => {
        console.error(`Failed to load audio file "${trial.audio_stimulus}". Try checking the file path. We recommend using the preload plugin to load audio files.`);
        console.error(err);
    });
  
    const setupAudioTrial =()=> {
        let html = ''; 
        // html += '<div id="jspsych-audio-audio-response-prompt">'+trial.prompt+'</div>';;
        //show prompt if there is one
        if (trial.prompt !== null) {
            html += trial.prompt;
        }
        display_element.innerHTML = html;
        // start time
        startTime = performance.now();
        var startTime;
        var context = this.jsPsych.pluginAPI.audioContext();
        // start audio
        if (context !== null) {
            startTime = context.currentTime;
            this.audio.start(startTime);
        }
        else {
            this.audio.play();
        }
        on_load();
        trial_complete();
    };
}
// on_load() {
//     // this.startRecording();
// }
  setupRecordingEvents(display_element, trial) {
      this.data_available_handler = (e) => {
          if (e.data.size > 0) {
              this.recorded_data_chunks.push(e.data);
          }
      };
      this.stop_event_handler = () => {
          const data = new Blob(this.recorded_data_chunks, { type: "audio/webm" });
          this.audio_url = URL.createObjectURL(data);
          const reader = new FileReader();
          reader.addEventListener("load", () => {
              const base64 = reader.result.split(",")[1];
              this.response = base64;
              this.load_resolver();
          });
          reader.readAsDataURL(data);
      };
      this.start_event_handler = (e) => {
          // resets the recorded data
          this.recorded_data_chunks.length = 0;
          this.recorder_start_time = e.timeStamp;

          if (trial.recording_duration !== null) {
              this.jsPsych.pluginAPI.setTimeout(() => {
                  // this check is necessary for cases where the
                  // done_button is clicked before the timer expires
                  if (this.recorder.state !== "inactive") {
                      this.stopRecording(display_element, trial).then(() => {
                          if (trial.allow_playback) {
                              this.showPlaybackControls(display_element, trial);
                          }
                          else {
                              this.endTrial(display_element, trial);
                          }
                      });
                  }
              }, trial.recording_duration);
          }
      };
      this.recorder.addEventListener("dataavailable", this.data_available_handler);
      this.recorder.addEventListener("stop", this.stop_event_handler);
      this.recorder.addEventListener("start", this.start_event_handler);
  }

  startRecording() {
      this.recorder.start();
  }


  stopRecording(display_element, trial) {
    this.recorder.stop();
    return new Promise((resolve) => {
      this.load_resolver = resolve;
      this.endTrial(display_element, trial); // Pass display_element and trial to endTrial
    });
  }
  
  endTrial(display_element, trial) {
    var context = this.jsPsych.pluginAPI.audioContext();
      // clear recordering event handler
      this.recorder.removeEventListener("dataavailable", this.data_available_handler);
      this.recorder.removeEventListener("start", this.start_event_handler);
      this.recorder.removeEventListener("stop", this.stop_event_handler);
      // kill any remaining setTimeout handlers
      this.jsPsych.pluginAPI.clearAllTimeouts();
    //   if (context !== null) {
    //     this.audio.stop();
    // }
    //   else {
    //     this.audio.pause();
    // }
    //   this.audio.removeEventListener("ended", this.endTrial);
    // this.audio.removeEventListener("ended", setup_keyboard_listener);
      // gather the data to store for the trial
      var trial_data = {
          rt: this.rt,
          stimulus: trial.audio_stimulus,
          response: this.response,
          estimated_stimulus_onset: Math.round(this.stimulus_start_time - this.recorder_start_time),
      };
      if (trial.save_audio_url) {
          trial_data.audio_url = this.audio_url;
      }
      else {
          URL.revokeObjectURL(this.audio_url);
      }
      // clear the display
      display_element.innerHTML = "";
      // move on to the next trial
      this.jsPsych.finishTrial(trial_data);
  }

}

AudioAudioResponsePlugin.info = info;

return AudioAudioResponsePlugin;

})(jsPsychModule);
`
Thank you in advance for your help!

Zifeng

[ZifengLIU143]

I removed some unused functions and it works!