audio instead of images for the free-sort task

[dusannikolic92]

Hello,

I am trying to have audio recordings instead of images in a free-sort task. That is, I am trying to have images, and when participants click on images, the audio recording will be played. These images/audios should be dragged into the box based on their similarity. Can I do this with the free-sort plugin? Any help on this is welcome.

My code so far:

var timeline = [];

var preload = {
    type: 'preload',
    auto_preload: true
  };
timeline.push(preload);


 var instructions1 = {
            type: 'instructions',
            pages: [`<p>Welcome to the experiment!</p>` + 
  `<p>Your task is to listen to the recordings and group them into 8 categories based on their similarity!</p>` ],
            show_clickable_nav: true,
              post_trial_gap: 1000
        };
        timeline.push(instructions1);

  var trial_1 = {
    type: 'free-sort',
    stimuli: ['img/AA.png','img/BB.png','img/CC.png','img/DD.png', 'img/FF.png', 'img/EE.png', 'img/GG.png','img/HH.png', 'img/II.png', 'img/JJ.png','img/KK.png','img/LL.png','img/MM.png', 'img/NN.png', 'img/OO.png', 'img/PP.png','img/QQ.png', 'img/RR.png', 'img/SS.png', 'img/TT.png', 'img/UU.png', 'img/VV.png', 'img/WW.png', 'img/XX.png', 'img/YY.png', 'img/ZZ.png','img/AB.png','img/CD.png','img/EF.png','img/GH.png', 'img/IJ.png', 'img/KL.png', 'img/MN.png','img/OP.png', 'img/QR.png'],
    stim_height: 25,
    stim_width: 70,
    sort_area_height: 600,
    sort_area_width: 600,
    sort_area_shape: "square",
    border_color_in: '#DCDCDC',
    border_color_out: 'red',
    scale_factor: 1.5,
    prompt:  'Please move the images inside of the square. Each image corresponds to one audio recording you were given to listen to. Please, group all the audio recordings into 8 groups by dragging and dropping the images inside of the square. 8 groups do not need to have equal number of members.'
  };
  timeline.push(trial_1);

var block0 = {
    type: 'survey-text',
    questions: [
      {prompt: '<p>Congrats! You completed the experiment! Thank you for participating.</p>'+
'<p>If you noticed anything about the experiment (e.g. the strategies that you used to complete the task or the problems you had, what was easy/difficult, etc.), feel free to write it here .</p>', placeholder: 'Experiment is very interesting..', rows:10, columns: 50}
    ]
        };
   timeline.push(block0);

  jsPsych.init({
    timeline: timeline,
  });

[jodeleeuw]

Hi @dusannikolic92,

You'll need to modify the plugin to do this. I have a vague memory of someone else building a plugin to do this kind of thing. This paper is what I was able to find through a search. It looks like it might be a good starting point.

Edit: Looks like @pwdonh is actively developing tools for this at https://github.com/pwdonh/audio_tokens. Looks very cool!

[dusannikolic92]

Hello, Josh.

Thank you very much for the response. I will figure something out!

--Dusan

[pwdonh]

Yes, the tools under https://github.com/pwdonh/audio_tokens support a lot of ways to sort and rate audio recordings. There's no way to pair images with the audio files though: each audio file is represented by a colored circle. Not sure whether it's easier to modify the jsPsych-freesort plugin or modify my plugin for your needs. Mine is based on d3: there might be a way to fill circles with an image instead of a color (this?: https://javascript.tutorialink.com/adding-an-image-within-a-circle-object-in-d3-javascript/), if you wanna try that.

[dusannikolic92]

Thank you!

Will take a look and let you know if I have some qs, but it looks good!

Best,
Dusan

[dusannikolic92]

Hello @jodeleeuw and @pwdonh,

I have been trying to adapt the experiment for jspsych, but I keep getting errors on the "new FeatureRatings()" as it is unrecognized, and on the 'graph.build()' function in the plugin.
I think this toolbox could be very well and frequently used with jspsych for various experiments, so it would be worth it spending a bit of time to figure out the errors. Thank you!

Here is the javascript code for the experiment:

var timeline = [];

    
 var imagine = {
   type: 'html-button-response',
   stimulus: '<p>Welcome to the experiment</p>',
 choices: ['Start']
 }
 
 timeline.push(imagine);

 var stimuli = [
            'data/cven_00_02286_84473_617520.wav',
            'data/cven_03_26059_84473_17388318.wav',
            'data/cven_05_14026_84473_17255724.wav',
            'data/cven_06_31405_84473_97708.wav',
            'data/cven_07_27838_84473_566483.wav',
            'data/cven_08_19497_84473_17246733.wav',
            'data/cven_09_26764_84473_17637624.wav',
            'data/cven_10_19680_84473_665433.wav',
            'data/cven_17_17551_84473_512103.wav'
        ]

        var single_feature_trial = {
            type: 'audio-tokens',
            ratingtype: 'features',
            stimuli: stimuli.slice(0,4), // select only first four stimuli
            label: ['Feature 1'],
            anchors: [['low', '', 'high']]
        }

        var multiple_feature_trial = {
            type: 'audio-tokens',
            ratingtype: 'features',
            stimuli: stimuli.slice(0,4),
            label: ['Feature 1', 'Feature 2', 'Feature 3'],
            anchors: [['low', '', 'high'],
                      ['low', '', 'high'],
                      ['low', '', 'high']]
        }

        var features2d_trial = {
            type: 'audio-tokens',
            ratingtype: 'features2d',
            stimuli: stimuli, // select all stimuli
            label: ['Feature 1', 'Feature 2'],
            anchors: [['low', '', 'high'],
                      ['low', '', 'high']]
        }

        var categories_trial = {
            type: 'audio-tokens',
            ratingtype: 'categories',
            stimuli: stimuli,
            label: ['Category 1', 'Category 2'],
        }

        var cluster_trial = {
            type: 'audio-tokens',
            ratingtype: 'cluster',
            stimuli: stimuli,
        }

        var similarity_trial = {
            type: 'audio-tokens',
            ratingtype: 'similarity',
            stimuli: stimuli,
        }

        var triplets_trial = {
            type: 'audio-tokens',
            ratingtype: 'triplets',
            stimuli: stimuli
        }

        // Add all trials to a timeline array
        var good = {
          timeline: [
            single_feature_trial,
            multiple_feature_trial,
            features2d_trial,
            categories_trial,
            cluster_trial,
            similarity_trial,
            triplets_trial
        ]
};
timeline.push(good);

        // Start jspsych experiment
        jsPsych.init({
            timeline: timeline,
            show_progress_bar: true,
            on_finish: function() {
                jsPsych.data.displayData();
            }
        })

Here is the audio-tokens plugin code:

/*
 * Example plugin template
 */

jsPsych.plugins["audio-tokens"] = (function() {

  var plugin = {};

  plugin.info = {
    name: "audio-tokens",
    parameters: {
      stimuli: {
        type: jsPsych.plugins.parameterType.OBJECT,
        default: ['file1.wav', 'file2.wav']
      },
      ratingtype: {
        type: jsPsych.plugins.parameterType.STRING, // BOOL, STRING, INT, FLOAT, FUNCTION, KEY, SELECT, HTML_STRING, IMAGE, AUDIO, VIDEO, OBJECT, COMPLEX
        default: 'cluster' // 'features', 'cluster' or 'similarity'
      },
      // Parameter for ratingtype 'features' and 'categories'
      label: {
        type: jsPsych.plugins.parameterType.OBJECT,
        default: ['Feature to rate']
        // for multiple features use:
        // ['1st Feature to rate', '2nd Feature to rate', '3rd Feature to rate']
      },
      // Parameter for ratingtype 'features'
      anchors: {
        type: jsPsych.plugins.parameterType.OBJECT,
        default: [['low', 'medium', 'high']]
        // for multiple features use:
        // [['low', 'medium', 'high'],
        //  ['low', 'medium', 'high'],
        //  ['low', 'medium', 'high']]
      },
      force_listen: {
        type: jsPsych.plugins.parameterType.BOOL,
        default: false
      }
    }
  }

  plugin.trial = function(display_element, trial) {

    if (trial.force_listen) {
      var opacity = .5
    } else {
      var opacity = 1.
    }
    var height = {'features': 150*trial.label.length,
                  'cluster': 400,
                  'similarity': 400,
                  'features2d': 375,
                  'categories': 225,
                  'triplets': 150}

    var html = ''

    html += '<div class="container" style="margin-bottom:25px">'
    html += '<div class="d-flex justify-content-center">'
    html += '<div id="button-container" class="btn-group" style="margin-bottom:25px">'
    html += '</div>'
    html += '</div>'
    html += '</div>'
    html += '<div class="container" style="margin-bottom:25px">'
    html += '<div class="d-flex justify-content-center">'
    html += '<div id="plot-speakers-div">'
    html += '<svg id="plot-speakers" width="400" height="'+height[trial.ratingtype]+'">'
    html += '</div>'
    html += '</div>'
    html += '<div id="audio-container"></div>'

    display_element.innerHTML = html

    var num_speakers = trial.stimuli.length
    data = {'nodes': []}
    for (i=0; i<num_speakers; i++) {
      data.nodes.push({'id': 'item-'+String(i), 'audiofile': trial.stimuli[i],
                       'x': [], 'y': []})
    }

    if (trial.ratingtype=='cluster') {
      var graph = new CircleSortGraph(data, 'plot-speakers', 'audio-container',
                                      buttonContainerId='button-container',
                                      draw_edges=true, trial_id='', width=400,
                                      nextURL=display_element, 
                                      opacity=opacity, isJsPsych=true)
    } else if (trial.ratingtype=='similarity') {
      var graph = new AudioGraph(data, 'plot-speakers', 'audio-container',
                                   buttonContainerId='button-container',
                                   draw_edges=true, trial_id='', width=400,
                                   nextURL=display_element, 
                                   opacity=opacity, isJsPsych=true)
    } else if (trial.ratingtype=='features') {
      var graph = new FeatureRatings(data, 'plot-speakers', 'audio-container',
                                     buttonContainerId='button-container',
                                     draw_edges=true,
                                     num_features=trial.label.length,
                                     feature_labels=trial.label,
                                     feature_anchors=trial.anchors,
                                     item_spacing=7.5, trial_id='', width=400,
                                     nextURL=display_element, 
                                     opacity=opacity, isJsPsych=true)
    } else if (trial.ratingtype=='features2d') {
      var graph = new FeatureRatings2D(data, 'plot-speakers', 'audio-container',
                                       buttonContainerId='button-container',
                                       draw_edges=true,
                                       feature_labels=trial.label,
                                       feature_anchors=trial.anchors,
                                       trial_id='', width=400,
                                       nextURL=display_element, 
                                       opacity=opacity, isJsPsych=true)
    } else if (trial.ratingtype=='categories') {
      var graph = new FreesortGraph(data, 'plot-speakers', 'audio-container',
                                    buttonContainerId='button-container',
                                    draw_edges=false, trial_id='', width=400,
                                    nextURL=display_element, 
                                    opacity=opacity, isJsPsych=true,
                                    num_col=trial.label.length,
                                    item_spacing=12.5,
                                    feature_labels=trial.label)
    } else if (trial.ratingtype=='triplets') {
      var graph = new TripletAudioGraph(data, 'plot-speakers', 'audio-container',
                                        buttonContainerId='button-container',
                                        draw_edges=true, trial_id='', width=400,
                                        nextURL=display_element, 
                                        opacity=opacity, isJsPsych=true)
    }

    graph.build()

  };

  return plugin;
})();

[pwdonh]

Have you checked that all of the necessary Javascript files are included?

In addition to jspsych, that would be:

    <script src="jspsych-audio-tokens.js"></script>
    <script src="//d3js.org/d3.v4.min.js"></script>
    <script src="audio-tokens.js"></script>

sounds like the last one might be missing.

Feel free to ask questions in my github repository (https://github.com/pwdonh/audio_tokens)