resampling at the second level of embedding

[y-j-huang]

Hi everyone, I created a practice structure where I have two nested levels (Level A and B). Each condition in Level A contains 6 conditions in Level B which contains 7 trials each). So there are 42 trials in total. I want to sample at level A (the upper level so that the relevant items in level B will be grouped together). I also want to record the accuracy so any participant whose accuracy is lower than 0.8 will need to redo the practice. If I sample with the sampling method, it only samples at level B for every individual item. So, I created an array at level A. In addition, I created a loop so that the practice will repeat if the accuracy is below 0.8 However, the loop doesn't update the sample, so the practice items stays the same while I want them to be different every time (I don't want toe participants to repeat the same practice items over again). Here's my code:

  var practice_trial = {
    type: 'survey-multi-choice',
    questions: jsPsych.timelineVariable('questions'),
        data:{
            trial_part: 'practice',
            questions: jsPsych.timelineVariable('questions'),
            correct_response: jsPsych.timelineVariable('correct_response')},
    on_finish: function(data) {
            var acc = false;
            if (data.correct_response == JSON.stringify(data.response)) {
                acc = true;
            }
            data.accuracy = acc;
        },
    };
 
  var feedback = {
   ...
    };
    
    var practice_1 = {
        timeline: [practice_trial, feedback],
        timeline_variables: verb1

    }
    
    var practice_2 = {
        timeline: [practice_trial, feedback],
        timeline_variables: verb2
    }
    
    var practice_3 = {
        timeline: [practice_trial, feedback],
        timeline_variables: verb2
    }
    
    var practice_3 = {
        timeline: [practice_trial, feedback],
        timeline_variables: verb3
    }
    
    var practice_4 = {
        timeline: [practice_trial, feedback],
        timeline_variables: verb4
    }
    
    var practice_5 = {
        timeline: [practice_trial, feedback],
        timeline_variables: verb5
    }
    
     var practice_6 = {
        timeline: [practice_trial, feedback],
        timeline_variables: verb6
    }
    
    var trials = [practice_1, practice_2, practice_3, practice_4, practice_5, practice_6];
    var sample = jsPsych.randomization.sampleWithoutReplacement(trials, 2);

// Repeat practice trial if too many mistakes

var repeat_prac_message = {
       ...
    };

    var repeat_prac_conditional = {
        timeline: [repeat_prac_message, instructions],
        conditional_function: function() {
            var last_prac_trials = jsPsych.data.get().filter({trial_part:'practice'}).last(14);
            var n_correct = last_prac_trials.filter({accuracy: true}).count();
            var prop_corr = n_correct/14;
            if (prop_corr < 0.8) {
                repeat_prac = true;
            return true;
            } else {
                repeat_prac = false;
                return false;
            }
        }
    }

  var practice = {
    timeline: sample,
}

  var instructions_prac = {
        timeline: [practice, repeat_prac_conditional],
        loop_function: function() {
            if (repeat_prac == true) {
                // sample = jsPsych.randomization.sampleWithoutReplacement(trials, 1);
                return true;
            } else {
                return false;
            }
        }
    }

timeline.push(instructions_prac); 

I 'm guessing the "sample" is defined as a global variable so it doesn't work but putting it within the loop function doesn't work either. I would appreciate some insight of what I should change or if there's a different way to accomplish this. Thank you!

[becky-gilbert]

Hi @y-j-huang, I'm not sure if this will work, but you could try changing your practice timeline into a function, so that the trials are re-sampled every time the practice node runs:

var practice = {
    timeline: function() {
        return jsPsych.randomization.sampleWithoutReplacement(trials, 2);
    }
}

If that doesn't work, then what I would do is just put [practice, feedback] in the practice timeline, and then use timeline_variables to combine the trial information from the two stimuli sets that were selected. I'm pretty sure you can make the timeline_variables parameter into a function.

var verbs = [verb1, verb2, verb3, verb4, verb5, verb6];

var practice = {
    timeline: [practice_trial, feedback],
    timeline_variables: function() {
        var sampled_stimuli = jsPsych.randomization.sampleWithoutReplacement(verbs, 2);
        // sampled_stimuli contains two separate arrays, so we need to combine them into a single array using .concat()
        return sampled_stimuli[0].concat(sampled_stimuli[1]);
    }
};

Do either of those methods work?

[y-j-huang]

Hi @becky-gilbert ,

Thank you for the suggestions. I tried both and unfortunately neither worked. The first one just skips all the practices and directly goes into the repeat_prac_message. Since there's no error in the console, I'm not sure what's happening.

For the second solution, I assume that I need to move one level down:

        var verbs = [verb1, verb2, verb3, verb4, verb5, verb6]
        var practice = {
            timeline: [practice_trial, feedback],
            timeline_variables: function() {
                var sampled_stimuli = jsPsych.randomization.sampleWithoutReplacement(verbs, 2);
                // sampled_stimuli contains two separate arrays, so we need to combine them into a single array using .concat()
                return sampled_stimuli[0].concat(sampled_stimuli[1]);
            }
        };

However, this gives me an error: Uncaught TypeError: Cannot read property 'questions' of undefined
Because the function got stuck there, I moved the lines outside to print the output:

        var sampled_stimuli = jsPsych.randomization.sampleWithoutReplacement(verbs, 2);
        console.log(sampled_stimuli[0].concat(sampled_stimuli[1]))

The output seems normal to me and it contains my combined stim sets with the question parameter, so I'm not sure what the problem is. Any insights? Thank you!

[becky-gilbert]

Sorry, the answer I gave you above is wrong! 🤦 I edited it to make it a little bit less wrong. I'll answer in a separate comment.

[becky-gilbert]

Ok, instead of my suggestion to use a dynamic parameter for timeline_variables, which I don't think is possible, I think you can use your whole stimuli set as timeline_variables and then use the custom sampling option to get a different subset of stimuli each time the practice is repeated.

The custom sampling function receives one argument, which is an array of trial indices from 0 to n-1, where n is the number of trials in your timeline_variables array. The function needs to return the trial indices that you want to be presented on that run through the trial procedure. We can use some randomization functions inside of this custom sampling function so that, each time the procedure runs, it'll re-sample the trials.

Here's how I did this. First, I added the set name to all of the trial objects in your trial info arrays (verb1, verb2, etc.). So for me it looks like this:

var verb1 = [
  {
    questions: [
      {prompt: 'verb1 question 1', options: options}
    ],
    correct_response: '1',
    set: 'verb1'  // label each trial with its set name
  },
  {
    questions: [
      {prompt: 'verb1 question 2', options: options}
    ],
    correct_response: '2',
    set: 'verb1'
  }
];
// same for verb2, verb3, etc.

Then I combined all of these into a single flattened array so that it can be passed to timeline_variables.

var verbs = verb1.concat(verb2).concat(verb3);  // etc.

I also created a list of the stimuli set names, called set_names, in order to randomly sample from these sets (rather than sampling from the whole list of individual trials).

var set_names = ['verb1','verb2','verb3'];  // etc.

Finally, in the practice procedure, I used the flattened array containing all of the trials (verbs) as the timeline_variables array, and did the sampling inside the custom sampling function:

var practice = {
  timeline: [practice_trial],
  timeline_variables: verbs,
  sample: {
    type: 'custom',
    fn: function(t) {
      // empty array to store the trials that will be presented
      var sampled_trials = [];
      // randomly select two stimuli set names (e.g. verb1, verb2, etc.)
      var sampled_sets = jsPsych.randomization.sampleWithoutReplacement(set_names,2);
      t.forEach(function(x) {
        // for each trial in the verbs array, check if its 'set' name is one of the sampled sets,
        // and if so, add the trial to the array of trials that will be presented
        if (sampled_sets.includes(verbs[t[x]].set)) {
          sampled_trials.push(t[x]);
        }
      });
      // randomize the order of the selected trials and return it
      return jsPsych.randomization.shuffle(sampled_trials);
    }
  }
};

One other thing that I noticed - when checking the participant's response against the correct response, the responses are stored in the response object, where there's one key per question, which is either the question name (if you gave it one), or Q0, Q1, etc. So in the practice_trial on_finish function, instead of this:

if (data.correct_response == JSON.stringify(data.response)) {

I checked the response accuracy like this (I only had one question per page and didn't name them):

if (data.response.Q0 == data.correct_response)

I hope this helps! Let us know if this solution works for you.

[y-j-huang]

@becky-gilbert Thank you! I figured it out. I originally put set in an embedder level which didn't work. Now it works. Is there a way to stop the study after the sampling happened for a certain amount of time? e.g. the study terminates after two rounds of practice trials.

[becky-gilbert]

HI @y-j-huang, yes, you can add the repetitions parameter to repeat the timeline and sampling procedure. To repeat it twice, you would do this:

var practice = {
  timeline: [practice_trial],
  timeline_variables: verbs,
  sample: {
    type: 'custom',
    fn: function(t) {
      // ...
    }
  },
  repetitions: 2
};

Keep in mind that, on each repetition, the sampling function will use the jsPsych.randomization.sampleWithoutReplacement function to randomly sample a certain number of verb sets from the whole list of set_names. This means that some trials could be repeated in the first and second repetitions of the timeline, if the same verb set(s) happen to be randomly selected both times. If this is a problem, one way to solve it is by removing the randomly-sampled sets from the set_names array inside the sample function, so that on the next repetition, you're only sampling from the sets that haven't been presented yet. This is one way to do that (inside the sample function):

// randomly select two stimuli set names (e.g. verb1, verb2, etc.)
var sampled_sets = jsPsych.randomization.sampleWithoutReplacement(set_names,2);
// get the indices for the two sampled sets in the set_names array
var set_index_1 = set_names.indexOf(sampled_sets[0]);
var set_index_2 = set_names.indexOf(sampled_sets[1]); 
// remove each of these set names from the set_names array, so that they can't be selected on the next repetition
set_names.splice(set_index_1,1);
set_names.splice(set_index_2,1);

I hope this helps. Let us know if this works for you.

[y-j-huang]

@becky-gilbert the repetition doesn't work as expected. Instead, it repeats every sampled verb set twice in each practice session. But the practice session continuous after being called twice. I tried to put the repetition in the loop function instead (instructions_prac) but that still doesn't work. In addition, I'd want to terminate the entire study not just the practice session (if a participant fails the practice trials twice, we'll need to reject that participant). Is there a way to do that? Thank you!

[becky-gilbert]

@y-j-huang can you please share the current version of your code, and I'll have a look. Thanks!

[y-j-huang]

@becky-gilbert here's the current version of my code:

var practice_trial = {
    type: 'survey-multi-choice',
    questions: jsPsych.timelineVariable('questions'),
        data:{
            trial_part: 'practice',
            questions: jsPsych.timelineVariable('questions'),
            correct_response: jsPsych.timelineVariable('correct_response')},
    on_finish: function(data) {
            var acc = false;
            if (data.correct_response == JSON.stringify(data.response)) {
                acc = true;
            }
            data.accuracy = acc;
        },
    };
 
var feedback = {
        type: 'html-keyboard-response',
        questions: jsPsych.timelineVariable('questions'),
        stimulus: function() {
            var resp = jsPsych.data.getLastTrialData().values()[0].response[0];
            var feedback_text = '<span style="color:red;font-size:30px;">Incorrect. It should be </span>'+ '<span style="font-size:30px;">'+jsPsych.timelineVariable('answer')+'</span>';
            var last_resp_acc = jsPsych.data.getLastTrialData().values()[0].accuracy;
            if (last_resp_acc == true) {
                feedback_text = '<span style="color:green;font-size:30px;">Correct!</span>'
            }
            return feedback_text;
        },
        choices: jsPsych.NO_KEYS,
        trial_duration: 2000
    };

var verbs = verb1.concat(verb2).concat(verb3).concat(verb4).concat(verb5).concat(verb6); 
console.log(verbs)
var set_names = ['verb1','verb2','verb3', 'verb4', 'verb5', 'verb6']; 

var practice = {
    timeline: [practice_trial],
    timeline_variables: verbs,
    sample: {
        type: 'custom',
        fn: function(t) {
        // empty array to store the trials that will be presented
            var sampled_trials = [];
        // randomly select two stimuli set names (e.g. verb1, verb2, etc.)
            var sampled_sets = jsPsych.randomization.sampleWithoutReplacement(set_names,2);
        // get the indices for the two sampled sets in the set_names array
            var set_index_1 = set_names.indexOf(sampled_sets[0]);
            var set_index_2 = set_names.indexOf(sampled_sets[1]); 
        // remove each of these set names from the set_names array, so that they can't be selected on the next repetition
            set_names.splice(set_index_1,1);
            set_names.splice(set_index_2,1);
            t.forEach(function(x) {
        // for each trial in the verbs array, check if its 'set' name is one of the sampled sets,
        // and if so, add the trial to the array of trials that will be presented
                if (sampled_sets.includes(verbs[t[x]].set)) {
                sampled_trials.push(t[x]);
                }
            });
     // randomize the order of the selected trials and return it
        return sampled_trials;
        }
    }  
};

// Repeat practice trial if too many mistakes

var repeat_prac_message = {
        type: 'html-button-response',
        stimulus: '<p>You did not make enough correct responses to continue.</p>'+
                  '<p>Now you will have a chance to repeat the instructions and practice trials again.</p>',
        choices: ['Next']
    };

    var repeat_prac_conditional = {
        timeline: [repeat_prac_message, instructions],
        conditional_function: function() {
            var last_prac_trials = jsPsych.data.get().filter({trial_part:'practice'}).last(14);
            var n_correct = last_prac_trials.filter({accuracy: true}).count();
            var prop_corr = n_correct/14;
            console.log(prop_corr)
            if (prop_corr < 0.8) {
                repeat_prac = true;
                return true;
            } else {
                repeat_prac = false;
                return false;
            }
        }
    }
    

 var instructions_prac = {
        timeline: [practice, repeat_prac_conditional],
        loop_function: function() {
            if (repeat_prac == true) {
                return true;
            } else {
                return false;
            }
        }
    }

timeline.push(instructions_prac);