llama_decode() crashes with 0xC0000005 (access-violation) on Windows when context grows; KV-cache helpers no longer available

[Mancee28]

Describe the bug
After a few hundred generated tokens llama_decode() aborts with

Process finished with exit code -1073741819 (0xC0000005)

No C++ exception is thrown; the crash happens inside the runtime (access to invalid address).

Expected behavior
llama_decode() should either:

• work indefinitely (automatic defragmentation), or
• return an error code if the caller must trim the KV-cache.

Actual behavior
Hard crash (Windows access-violation).

Environment

• llama.cpp commit: <SHA>
• OS: Windows 10
• Compiler: MSVC 17.11 / CMake 3.29
• Model: Mistral-7B-Instruct-v0.3-Q6_K.gguf
• GPU offload: off (CPU only)

Code
LLMInference.cpp:

#include "LLMInference.h"
#include <cstring>

#include "common.h"

void LLMInference::loadModel(const std::string& model_path, float min_p, float temperature) {
    // create an instance of llama_model
    llama_model_params model_params = llama_model_default_params();
    _model = llama_model_load_from_file(model_path.data(), model_params);

    if (!_model) {
        throw std::runtime_error("load_model() failed");
    }

    // create an instance of llama_context
    llama_context_params ctx_params = llama_context_default_params();
    ctx_params.n_ctx = 0;               // take context size from the model GGUF file
    ctx_params.no_perf = true;          // disable performance metrics
    ctx_params.defrag_thold = 0.1f;     // trigger defragmentation when 10% of context is fragmented
    _ctx = llama_init_from_model(_model, ctx_params);

    if (!_ctx) {
        throw std::runtime_error("llama_new_context_with_model() returned null");
    }

    // initialize sampler
    llama_sampler_chain_params sampler_params = llama_sampler_chain_default_params();
    sampler_params.no_perf = true;      // disable performance metrics
    _sampler = llama_sampler_chain_init(sampler_params);
    llama_sampler_chain_add(_sampler, llama_sampler_init_min_p(min_p, 1));
    llama_sampler_chain_add(_sampler, llama_sampler_init_temp(temperature));
    llama_sampler_chain_add(_sampler, llama_sampler_init_dist(LLAMA_DEFAULT_SEED));

    _formattedMessages = std::vector<char>(llama_n_ctx(_ctx));
    _messages.clear();

    _prevLen = 0;
    _nPast   = 0;
    _batch   = llama_batch_init(llama_n_ctx(_ctx), 0, 1);   // capienza massima = n_ctx
}

LLMInference::~LLMInference() {
    // libera le copie create con strdup()
    for (llama_chat_message &msg : _messages) {
        free((void*)msg.role);     // <-- era assente
        free((void*)msg.content);  // <-- usa free, non delete
    }

    llama_sampler_free(_sampler);
    llama_free(_ctx);
    llama_model_free(_model);
    llama_batch_free(_batch);      // libera il batch
}

void LLMInference::stopCompletion() {
    _prevLen = llama_chat_apply_template(
            nullptr,              // tmpl - usa il template di default del modello
            _messages.data(),     // chat
            _messages.size(),     // n_msg
            false,               // add_ass
            nullptr,             // buf
            0                    // length
    );
    if (_prevLen < 0) {
        throw std::runtime_error("llama_chat_apply_template() in LLMInference::stop_completion() failed");
    }
}

void LLMInference::startCompletion(const std::string& query) {
    addChatMessage(query, "user");

    // apply the chat-template
    int new_len = llama_chat_apply_template(
            nullptr,                    // tmpl - usa il template di default del modello
            _messages.data(),           // chat
            _messages.size(),           // n_msg
            true,                      // add_ass
            _formattedMessages.data(), // buf
            _formattedMessages.size()  // length
    );
    if (new_len > static_cast<int>(_formattedMessages.size())) {
        // resize the output buffer `_formattedMessages`
        // and re-apply the chat template
        _formattedMessages.resize(new_len);
        new_len = llama_chat_apply_template(nullptr, _messages.data(), _messages.size(), true, _formattedMessages.data(), _formattedMessages.size());
    }
    if (new_len < 0) {
        throw std::runtime_error("llama_chat_apply_template() in LLMInference::start_completion() failed");
    }
    std::string prompt(_formattedMessages.begin() + _prevLen, _formattedMessages.begin() + new_len);

    // tokenization - usa _ctx invece di _model
    _promptTokens = common_tokenize(_ctx, prompt, true, true);

    // (ri)inizializza il batch alle dimensioni effettive
    llama_batch_free(_batch);
    _batch = llama_batch_init(static_cast<int32_t>(_promptTokens.size()), 0, 1);

    for (int i = 0; i < static_cast<int>(_promptTokens.size()); ++i) {
        _batch.token   [i]      = _promptTokens[i];
        _batch.pos     [i]      = _nPast + i;   // posizione nel contesto
        _batch.n_seq_id[i]      = 1;            // una sola sequenza
        _batch.seq_id  [i][0]   = 0;            // id = 0
        _batch.logits  [i]      = (i == _promptTokens.size() - 1); // logits solo ultimo
    }
    _batch.n_tokens = static_cast<int32_t>(_promptTokens.size());
    _nPast += _batch.n_tokens;
}

std::string LLMInference::completionLoop() {
    // run the model
    if (llama_decode(_ctx, _batch) < 0) {
        throw std::runtime_error("llama_decode() failed");
    }

    // sample a token and check if it is an EOG (end of generation token)
    // convert the integer token to its corresponding word-piece
    _currToken = llama_sampler_sample(_sampler, _ctx, -1);

    // Ottieni il vocab dal modello
    const struct llama_vocab * vocab = llama_model_get_vocab(_model);
    if (llama_vocab_is_eog(vocab, _currToken)) {
        addChatMessage(strdup(_response.data()), "assistant");
        _response.clear();
        return "[EOG]";
    }
    std::string piece = common_token_to_piece(_ctx, _currToken, true);

    // prepara il batch con il token generato
    _batch.token   [0]    = _currToken;
    _batch.pos     [0]    = _nPast;
    _batch.n_seq_id[0]    = 1;
    _batch.seq_id  [0][0] = 0;
    _batch.logits  [0]    = 1;
    _batch.n_tokens       = 1;
    _nPast += 1;

    return piece;
}

void LLMInference::addChatMessage(const std::string& message, const std::string& role) {
    llama_chat_message chatMsg;
    chatMsg.role = strdup(role.c_str());
    chatMsg.content = strdup(message.c_str());
    _messages.push_back(chatMsg);
}

LLMInference.h:

#include <iostream>
#include <string>
#include <chrono>
#include <thread>
#include "LLMInference.h"

int main() {
    try {
        // Inizializza l'istanza di LLMInference
        LLMInference llm;

        // Carica il modello (sostituisci con il percorso del tuo modello)
        std::string model_path = model\Mistral-7B-Instruct-v0.3-Q6_K.gguf";
        float min_p = 0.1f;
        float temperature = 0.7f;

        std::cout << "Caricamento modello: " << model_path << std::endl;
        llm.loadModel(model_path, min_p, temperature);
        std::cout << "Modello caricato con successo!" << std::endl;

        // Loop principale per la chat
        std::string user_input;
        std::cout << "\n=== Chat con LLM ===\n";
        std::cout << "Digita 'quit' per uscire\n\n";

        while (true) {
            // Leggi input dall'utente
            std::cout << "Tu: ";
            std::getline(std::cin, user_input);

            // Controlla se l'utente vuole uscire
            if (user_input == "quit" || user_input == "exit") {
                break;
            }

            if (user_input.empty()) {
                continue;
            }

            // Inizia la generazione
            llm.startCompletion(user_input);

            std::cout << "Assistente: ";
            std::string response;
            std::string piece;

            // Loop di generazione token per token
            while (true) {
                piece = llm.completionLoop();

                // Controlla se abbiamo raggiunto la fine della generazione
                if (piece == "[EOG]") {
                    break;
                }

                // Stampa il token generato
                std::cout << piece << std::flush;
                response += piece;

                // Piccola pausa per rendere visibile la generazione
                std::this_thread::sleep_for(std::chrono::milliseconds(10));
            }

            // Ferma la generazione
            llm.stopCompletion();
            std::cout << std::endl << std::endl;
        }

        std::cout << "Arrivederci!" << std::endl;
        
    } catch (const std::exception& e) {
        std::cerr << "Errore: " << e.what() << std::endl;
        return 1;
    }
    
    return 0;
}

main.cpp:

#include <iostream>
#include <string>
#include <chrono>
#include <thread>
#include "LLMInference.h"

int main() {
    try {
        // Inizializza l'istanza di LLMInference
        LLMInference llm;

        // Carica il modello (sostituisci con il percorso del tuo modello)
        std::string model_path = "model\Mistral-7B-Instruct-v0.3-Q6_K.gguf";
        float min_p = 0.1f;
        float temperature = 0.7f;

        std::cout << "Caricamento modello: " << model_path << std::endl;
        llm.loadModel(model_path, min_p, temperature);
        std::cout << "Modello caricato con successo!" << std::endl;

        // Loop principale per la chat
        std::string user_input;
        std::cout << "\n=== Chat con LLM ===\n";
        std::cout << "Digita 'quit' per uscire\n\n";

        while (true) {
            // Leggi input dall'utente
            std::cout << "Tu: ";
            std::getline(std::cin, user_input);

            // Controlla se l'utente vuole uscire
            if (user_input == "quit" || user_input == "exit") {
                break;
            }

            if (user_input.empty()) {
                continue;
            }

            // Inizia la generazione
            llm.startCompletion(user_input);

            std::cout << "Assistente: ";
            std::string response;
            std::string piece;

            // Loop di generazione token per token
            while (true) {
                piece = llm.completionLoop();

                // Controlla se abbiamo raggiunto la fine della generazione
                if (piece == "[EOG]") {
                    break;
                }

                // Stampa il token generato
                std::cout << piece << std::flush;
                response += piece;

                // Piccola pausa per rendere visibile la generazione
                std::this_thread::sleep_for(std::chrono::milliseconds(10));
            }

            // Ferma la generazione
            llm.stopCompletion();
            std::cout << std::endl << std::endl;
        }

        std::cout << "Arrivederci!" << std::endl;
        
    } catch (const std::exception& e) {
        std::cerr << "Errore: " << e.what() << std::endl;
        return 1;
    }
    
    return 0;
}