How do I pass a list of tensors in onnxruntime-web?

[Meehai]

I have exported my LSTM-based model from python. I can call it using:

import onnxruntime as ort
import numpy as np
from scipy.special import softmax
bs = 2 # batch size
T = [6, 30] # sequence size

ort_session = ort.InferenceSession("model.onnx")
users = np.random.randn(bs, 10).astype(np.float32)
sessions = [np.random.randn(T[0], 117).astype(np.float32)], np.random.randn(T[1], 117).astype(np.float32)]
y_onnx = ort_session.run(None, {"users": users, "sessions": sessions})[0]
y_onnx = softmax(y_onnx, axis=-1)
print(y_onnx)

This outputs:

[[0. 0. 0. 1.]]

The sessionsis a List of np.ndarrays of shape (6, 117). How do I pass a list in onnxruntime-web?

What I've tried so far:

const ort = require('onnxruntime-web');
const Tensor = ort.Tensor;

// use an async context to call onnxruntime functions.
async function main() {
    try {
        const session = await ort.InferenceSession.create('model.onnx');
        document.write("loaded model<br/>");

        const dataUser = new Float32Array([0.8610470294952393, 0.04083799943327904, 0.15420299768447876, 0.17481200397014618, 0.0911799967288971, 1.0, 0.0, 0.2857140004634857, 1.0, 1.0])
       // some random data
        const dataSession = new Float32Array([0.0, 0.0, 0.008696000091731548, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0018810000037774444, 0.006802999880164862, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06086999922990799, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11382900178432465, 0.04761900007724762, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.24347800016403198, 0.928600013256073, 0.0, 0.0, 0.0, 0.0, 0.0714000016450882, 0.0, 0.0, 0.0, 0.0, 0.0714000016450882, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.1429000049829483, 0.8571000099182129, 0.0, 0.0, 0.1429000049829483, 0.8571000099182129, 0.0, 0.0, 0.8571000099182129, 0.0, 0.1429000049829483, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.2857140004634857, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.47999998927116394, 0.5, 0.270781010389328, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.8347830176353455, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.9090999960899353, 0.0, 0.2727000117301941, 0.6363999843597412, 0.0908999964594841, 0.0, 0.0, 0.3635999858379364, 0.6363999843597412, 0.0, 0.2727000117301941, 0.0, 0.6363999843597412, 0.0908999964594841, 0.0, 0.0, 0.0, 0.6363999843597412, 0.3635999858379364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008360999636352062, 1.0, 0.8163269758224487, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.800000011920929, 0.8333330154418945, 0.564844012260437, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3739129900932312, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.33329999446868896, 0.666700005531311, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.3781749904155731, 0.33333298563957214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.11999999731779099, 0.125, 0.322968989610672, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006688999943435192, 0.0, 0.006802999880164862, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.03999999910593033, 0.04166699945926666, 0.296875, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
       
        const tensorA = new Tensor('float32', dataUser, [2, 10]);
        document.write("tensor a done<br/>");
        // make a list here via []
        const tensorB = [new Tensor('float32', dataSession, [6, 117]), new Tensor('float32', dataSession, [6, 117])];
        document.write("tensor b done<br/>");

        const feeds = { users: tensorA, sessions: tensorB};

        // feed inputs and run
        document.write("before call<br/>");
        const results = await session.run(feeds);

        const dataC = results.c.data;
        document.write(`data of result tensor 'c': ${dataC}`);

    } catch (e) {
        document.write(`failed to inference ONNX model: ${e}.`);
    }
}

main();

I get the output:

loaded model
tensor a done
tensor b done
before call
failed to inference ONNX model: TypeError: i is undefined.

[Meehai]

model.zip
If anyone is interested, this is the model.onnx file that causes the issue. It works well with the python script, but can't make it do the same thing in js.

[fs-eire]

It looks like the input type of "sessions" is sequence:

unfortunately onnxruntime-web does not support non-tensor type input/output yet. contribution is welcome!

workaround: re-export the model with flags of disabling non-tensor types.

[Meehai]

I see, unfortunetely it has to be a list, or more specifically, a tensor of shape:sessions::shape = (bs, ?, n_feats). Therfore, sessions[0] has t0, n_feats items in the sequence, sessions[1] has t1, n_feats and so on.

I pad the entire batch (provided as a list of tensors in python) with zeros and pass through an lstm.

My workaround so far was to only accept batches of size 1 and provide the two tensors as users=(1, n_feats) and sessions=(T, n_feats) and T is sequence size, and it gets passed okay through the LSTM without any padding. However, batch_size=1 may hurt performance by a lot.

For my use-case it's okay, since I'll use a pre-trained pytorch model and just do inference in the onnxruntime-web, however, I can see this being an issue in the long run, especially if it's a supported operator (onnxruntime package in python accepts it just fine, it's the js implementation that doesn't).