added logprobs api and logprobs viewer

Author: LostRuins

expose.cpp

@@ -294,11 +294,29 @@ extern "C"
         return output;
     }
 
+    static std::vector<TopPicksData> last_logprob_toppicks;
+    static std::vector<logprob_item> last_logprob_items;
     last_logprobs_outputs last_logprobs()
     {
         last_logprobs_outputs output;
-        std::vector<TopPicksData> toppicks = gpttype_get_top_picks_data(); //copy top picks
-        output.count = 0;
+        last_logprob_items.clear();
+        last_logprob_toppicks.clear();
+        last_logprob_toppicks = gpttype_get_top_picks_data(); //copy top picks
+        for(int i=0;i<last_logprob_toppicks.size();++i)
+        {
+            logprob_item itm;
+            itm.option_count = last_logprob_toppicks[i].tokenid.size();
+            itm.selected_token = last_logprob_toppicks[i].selected_token.c_str();
+            itm.selected_logprob = last_logprob_toppicks[i].selected_logprob;
+            itm.logprobs = last_logprob_toppicks[i].logprobs.data();
+            for(int j=0;j<itm.option_count && j<logprobs_max;++j)
+            {
+                itm.tokens[j] = last_logprob_toppicks[i].tokens[j].c_str();
+            }
+            last_logprob_items.push_back(itm);
+        }
+        output.count = last_logprob_items.size();
+        output.logprob_items = last_logprob_items.data();
         return output;
     }
 

expose.h

@@ -3,6 +3,7 @@
 
 const int tensor_split_max = 16;
 const int images_max = 4;
+const int logprobs_max = 5;
 
 // match kobold's sampler list and order
 enum samplers
@@ -111,19 +112,26 @@ struct generation_outputs
 {
     int status = -1;
     int stopreason = stop_reason::INVALID;
+    int prompt_tokens = 0;
+    int completion_tokens = 0;
     const char * text; //response will now be stored in c++ allocated memory
 };
 struct token_count_outputs
 {
     int count = 0;
     int * ids; //we'll just use shared memory for this one, bit of a hack
 };
+
+struct logprob_item {
+    int option_count;
+    const char * selected_token;
+    float selected_logprob;
+    const char * tokens[logprobs_max];
+    float * logprobs = nullptr;
+};
 struct last_logprobs_outputs {
     int count = 0;
-    char ** selected_token;
-    float * selected_logprob;
-    char * tokens[5];
-    float * logprobs[5];
+    logprob_item * logprob_items = nullptr;
 };
 struct sd_load_model_inputs
 {

gpttype_adapter.cpp

@@ -597,13 +597,13 @@ llama_token sample_token(llama_token_data_array * candidates, std::mt19937 & rng
     int idx = dist(rng);
 
     newpick.selected_token = FileFormatTokenizeID(candidates->data[idx].id, file_format, true);
-    newpick.selected_logprob = candidates->data[idx].logit;
+    newpick.selected_logprob = logf(candidates->data[idx].p);
     newpick.selected_probability = candidates->data[idx].p;
     newpick.selected_tokenid = candidates->data[idx].id;
-    for (size_t i = 0; (i < candidates->size && i<5); ++i)
+    for (size_t i = 0; (i < candidates->size && i<logprobs_max); ++i)
     {
         newpick.tokens.push_back(FileFormatTokenizeID(candidates->data[i].id, file_format, true));
-        newpick.logprobs.push_back(candidates->data[i].logit);
+        newpick.logprobs.push_back(logf(candidates->data[i].p));
         newpick.p.push_back(candidates->data[i].p);
         newpick.tokenid.push_back(candidates->data[i].id);
     }
@@ -2467,6 +2467,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
         printf("\nWarning: KCPP text generation not initialized!\n");
         output.text = nullptr;
         output.status = 0;
+        output.prompt_tokens = output.completion_tokens = 0;
         output.stopreason = stop_reason::INVALID;
         generation_finished = true;
         return output;
@@ -3142,6 +3143,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
                 fprintf(stderr, "\nFailed to predict at %d! Check your context buffer sizes!\n",n_past);
                 output.text = nullptr;
                 output.status = 0;
+                output.prompt_tokens = output.completion_tokens = 0;
                 output.stopreason = stop_reason::INVALID;
                 generation_finished = true;
                 return output;
@@ -3471,6 +3473,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
                                 fprintf(stderr, "\nFailed to eval llava image at %d!\n",n_past);
                                 output.text = nullptr;
                                 output.status = 0;
+                                output.prompt_tokens = output.completion_tokens = 0;
                                 output.stopreason = stop_reason::INVALID;
                                 generation_finished = true;
                                 return output;
@@ -3482,6 +3485,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
                             fprintf(stderr, "\nLLAVA image tokens mismatch at %d! (%d vs %d tokens)\n",n_past,llavatokenscounted,llavatokensevaled);
                             output.text = nullptr;
                             output.status = 0;
+                            output.prompt_tokens = output.completion_tokens = 0;
                             output.stopreason = stop_reason::INVALID;
                             generation_finished = true;
                             return output;
@@ -3534,6 +3538,9 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
     printf("\nCtxLimit:%d/%d, Amt:%d/%d, Init:%.2fs, Process:%.2fs (%.1fms/T = %.2fT/s), Generate:%.2fs (%.1fms/T = %.2fT/s), Total:%.2fs (%.2fT/s)",(int)current_context_tokens.size(),(int)nctx, realnpredict, kcpp_data->n_predict, time0, time1, pt1, ts1, time2, pt2, ts2, (time1 + time2), tokens_per_second);
     fflush(stdout);
     output.status = 1;
+    int finaltokcount = (int)current_context_tokens.size()-realnpredict;
+    output.prompt_tokens = (finaltokcount<0?0:finaltokcount);
+    output.completion_tokens = realnpredict;
     output.stopreason = last_stop_reason;
     last_eval_time = pt2;
     last_process_time = pt1;