maby, it is A bug of TensorRT, ms_deform_attn_im2col_bilinear in plugin/multiscaleDeformableAttnPlugin/multiscaleDeformableIm2ColCuda.cuh

[boomWu]

Description

As the code from line 206 to line 241, for example , if h=-2.1 w=-2.1, then v1=0, v2=0, v3=3, v4 is uncertain and out of range

int32_t const hLow = floor(h);
int32_t const wLow = floor(w);
int32_t const hHigh = hLow + 1;
int32_t const wHigh = wLow + 1;
scalar_t v1 = 0;
if (hLow >= 0 && wLow >= 0)
{
    int32_t const ptr1 = hLowPtrOffset + wLowPtrOffset + basePtr;
    v1 = bottomData[ptr1];
    gradHWeight -= hw * v1;
    gradWWeight -= hh * v1;
    atomicAdd(gradValue + ptr1, w1 * topGradvalue);
}
scalar_t v2 = 0;
if (hLow >= 0 && wHigh <= width - 1)
{
    int32_t const ptr2 = hLowPtrOffset + wHighPtrOffset + basePtr;
    v2 = bottomData[ptr2];
    gradHWeight -= lw * v2;
    gradWWeight += hh * v2;
    atomicAdd(gradValue + ptr2, w2 * topGradvalue);
}
scalar_t v3 = 0;
if (hHigh <= height - 1 && wLow >= 0)
{
    int32_t const ptr3 = hHighPtrOffset + wLowPtrOffset + basePtr;
    v3 = bottomData[ptr3];
    gradHWeight += hw * v3;
    gradWWeight -= lh * v3;
    atomicAdd(gradValue + ptr3, w3 * topGradvalue);
}
scalar_t v4 = 0;
if (hHigh <= height - 1 && wHigh <= width - 1)
{
    int32_t const ptr4 = hHighPtrOffset + wHighPtrOffset + basePtr;
    v4 = bottomData[ptr4];
    gradHWeight += lw * v4;
    gradWWeight += lh * v4;
    atomicAdd(gradValue + ptr4, w4 * topGradvalue);
}

[boomWu]

h=hlm, w=wlm, they are limited to [-1,-1] [spatialH,spatialW], so it not a bug

            if (hIm > -1 && wIm > -1 && hIm < spatialH && wIm < spatialW)
            {
                col += ms_deform_attn_im2col_bilinear(
                           dataValuePtr, spatialH, spatialW, numHeads, channels, hIm, wIm, mCol, cCol)
                    * weight;
            }