[BUG]: decode: cannot decode batches with this context (calling encode() instead)

[aropb]

Description

Reranker error:
decode: cannot decode batches with this context (calling encode() instead)
D:\a\LLamaSharp\LLamaSharp\ggml\src\ggml-cpu\ops.cpp:5116: GGML_ASSERT(i01 >= 0 && i01 < ne01) failed

Model: jina-reranker-v2-base-multilingual-FP16.gguf

Now, when this error occurs, the application crashes.

I don't understand how to fix it yet. The error appears on some datasets, but not always.

This error is mentioned here:
ggml-org/llama.cpp#14454

[aropb]

I understood what was going on, we need to control and trim the tokens in Rerankers according to the length of the model context.

This my simple Reranker class

using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using LLama;
using LLama.Abstractions;
using LLama.Exceptions;
using LLama.Native;

namespace xxxxxx;

public sealed class LLamaTextReranker : IDisposable
{
public readonly LLamaContext Context;

public LLamaTextReranker(LLamaWeights weights, IContextParams @params, ILogger logger = null)
{
    if (@params.UBatchSize != @params.BatchSize)
        throw new ArgumentException("For non-causal models, batch size must be equal to ubatch size", nameof(@params));

    if (weights.NativeHandle is { HasEncoder: true, HasDecoder: true })
        throw new NotSupportedException("Computing rank in encoder-decoder models is not supported");

    if (@params.PoolingType != LLamaPoolingType.Rank)
        throw new NotSupportedException("Computing rank score, PoolingType must be equal to LLamaPoolingType.Rank");

    Context = weights.CreateContext(@params, logger);
    NativeApi.llama_set_embeddings(Context.NativeHandle, true);
}

public void Dispose()
{
    Context.Dispose();
}

public async Task<float> GetRelevance(LLamaToken[] inputTokens, string document, CancellationToken cancellationToken = default)
{
    Context.NativeHandle.MemoryClear();
    var docTokens = Tokenize(document);

    LLamaToken[] tokens = [.. inputTokens, .. docTokens];
    LLamaBatch batch = new();

    for (var i = 0; i < Math.Min(tokens.Length, Context.ContextSize); i++)
    {
        batch.Add(tokens[i], i, LLamaSeqId.Zero, true);
    }

    var result = await Context.DecodeAsync(batch, cancellationToken);

    if (result != DecodeResult.Ok)
    {
        throw new RuntimeError($"Failed to decode: {result}");
    }

    float score = Context.NativeHandle.GetEmbeddingsSeq(LLamaSeqId.Zero)[0];
    return Sigmoid(score) * 100f;
}

public async Task<float> GetRelevance(string input, string document, CancellationToken cancellationToken = default)
{
    var inputTokens = Tokenize(input);
    return await GetRelevance(inputTokens, document, cancellationToken);
}

public LLamaToken[] Tokenize(string input)
{
    return Context.Tokenize(text: input, addBos: true, special: true);
}

private static float Sigmoid(float x)
{
    return (float)(1f / (1f + Math.Exp(-x)));
}

}