Latest commits, manual biasing

* added `logit_bias_strings_manual` for real-time shifting of unwanted tokens
* minor fixes

Author: MaggotHATE

Makefile

@@ -424,8 +424,8 @@ HEADERS_GGUF_BASE = \
     $(ggmlsrc_f_s)/ggml-threading.h \
     $(ggmlsrc_cpu_f)/hbm.h \
     $(ggmlsrc_cpu_f)/ggml-cpu-impl.h \
-    $(ggmlsrc_cpu_f)/ggml-cpu-quants.h \
-    $(ggmlsrc_cpu_f)/ggml-cpu-traits.h \
+    $(ggmlsrc_cpu_f)/quants.h \
+    $(ggmlsrc_cpu_f)/traits.h \
     $(ggmlsrc_cpu_f)/common.h \
     $(ggmlsrc_cpu_f)/binary-ops.h \
     $(ggmlsrc_cpu_f)/unary-ops.h \
@@ -751,7 +751,7 @@ ui_simple = $(uibackend_f)/UI_simple.h
 endif
 
 # Final parts
-$(TMP)$(PREFIX)_class_chat.o:$(conapp) $(COMMON_H_DEPS) $(json_layer) $(chat_layer) $(settings_layer) $(OBJS_GGUF)
+$(TMP)$(PREFIX)_class_chat.o:$(conapp) $(HEADERS_GGUF_BASE) $(COMMON_H_DEPS) $(json_layer) $(chat_layer) $(settings_layer) $(OBJS_GGUF)
 	@echo ------------------------------------------------------------------------
 	$(CXX) $(I_GGUF) $(CXXFLAGS) $(LDFLAGS) -c $< -o $@
 	@echo ---------------CHAT COMPILED with: $(PREFIX)

base_sampling2/chat_layer.h

@@ -291,9 +291,12 @@ class chat
     std::string logit_bias_strings_display = "";
     std::string logit_bias_strings_ext_display = "";
     std::string logit_bias_strings_start_display = "";
+    std::string logit_bias_strings_manual_display = "";
 
     std::string last_candidates_logits_display = "";
 
+    std::string dry_sequence_breakers_display = "";
+
     struct llama_perf_context_data ctx_performance_data;
 
     //std::map<std::string,std::string> stats;
@@ -601,6 +604,57 @@ class chat
         // std::getline(std::cin, pause);
     }
 
+    void sparams_postfill2() {
+        // std::string space = " ";
+        if (params.sparams.logit_bias_strings_manual.size()) {
+            for (llama_token i = 0; i < llama_vocab_n_tokens(vocab); i++) {
+                std::string token_str = common_token_to_piece(ctx, i);
+                // cutting spaces since there are "duplicated" tokens with them
+                if (token_str.front() == ' ') {
+                    token_str = token_str.substr(1);
+                }
+
+                // almost never happens
+                if (token_str.back() == ' ') {
+                    token_str.pop_back();
+                }
+
+                bool restricted = false;
+                float bias = -INFINITY;
+
+                if (token_str.length() > 2) {
+                    for (auto word : params.sparams.logit_bias_strings_manual) {
+                        auto token_str_pos = word.find(token_str);
+
+                        if (token_str_pos == 0 || token_str_pos == (word.size() - 1)) {
+                            restricted = true;
+                            break;
+                        } else if (token_str.find(word) == 0 && (token_str.length() - word.length()) < 4) {
+                            restricted = true;
+                            break;
+                        }
+                    }
+                } else if (token_str.length() > 0) {
+                    for (auto word : params.sparams.logit_bias_strings_manual) {
+                        if (token_str == word) {
+                            restricted = true;
+                            break;
+                        }
+                    }
+                }
+
+                if (restricted == true) {
+                    params.sparams.logit_bias_tokens_manual.push_back(i);
+                }
+            }
+        }
+
+        // std::string pause = "";
+        // std::getline(std::cin, pause);
+    }
+
+
+
     bool logit_bias_check_exact(std::string_view token_str) {
         for (auto word : params.sparams.logit_bias_strings_exact) {
             if (token_str == word) return true;
@@ -757,6 +811,7 @@ class chat
         logit_bias_strings_display = "";
         logit_bias_strings_ext_display = "";
         logit_bias_strings_start_display = "";
+        logit_bias_strings_manual_display = "";
 
         for (auto l : params.sparams.logit_bias) {
             if (l.bias == -INFINITY) { 
@@ -769,6 +824,10 @@ class chat
         for (auto l : logit_bias_tokens_start) {
             logit_bias_strings_start_display += std::format(" '{}';", common_token_to_piece(ctx, l));
         }
+
+        for (auto l : params.sparams.logit_bias_tokens_manual) {
+            logit_bias_strings_manual_display += std::format(" '{}';", common_token_to_piece(ctx, l));
+        }
     }
 
     void get_last_candidates_logits_display() {
@@ -779,6 +838,14 @@ class chat
         }
     }
 
+    void get_dry_sequence_breakers_display() {
+        dry_sequence_breakers_display.clear();
+
+        for (auto breaker : params.sparams.dry_sequence_breakers) {
+            dry_sequence_breakers_display += std::format("{}; ", breaker);
+        }
+    }
+
     void params_postfill() {
         if (params.kv_overrides_pair.size()) kv_override_prefill();
         common_process_override_tensors(params);
@@ -1296,11 +1363,12 @@ class chat
         printf("%s: llama_n_ctx = %d\n", __func__, n_ctx);
 
         // processing restricted words into logit_bias
-        // sparams_postfill();
+        sparams_postfill2();
         //sparams_postfill_ext();
         // get_safeguard_token("Title");
         processByVocab("Title");
-
+        get_logit_bias_str();
+        get_dry_sequence_breakers_display();
 
         smpl = common_sampler_init(model, sparams);
         printf("%s: common_sampler_init\n", __func__);
@@ -1611,6 +1679,7 @@ class chat
     void check_antiprompt_tkns() {
         // check for reverse prompt using special tokens
         llama_token last_token = common_sampler_last(smpl);
+
         for (std::vector<llama_token> ids : antiprompt_ids) {
             if (std::size(ids) == 1 && last_token == ids[0]) {
                 if (params.interactive) {
@@ -1623,6 +1692,24 @@ class chat
         }
     }
 
+    bool check_antiprompt_tkns_bool() {
+        // check for reverse prompt using special tokens
+        llama_token last_token = common_sampler_last(smpl);
+
+        for (std::vector<llama_token> ids : antiprompt_ids) {
+            if (std::size(ids) == 1 && last_token == ids[0]) {
+                if (params.interactive) {
+                    is_interacting = true;
+                    has_antiprompt = std::format("{}: already has antiprompt", __func__);
+                }
+                is_antiprompt = true;
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     //checking already existing contex
     int checkEmbd(){
         if (debug) printf("-ce");
@@ -1678,15 +1765,19 @@ class chat
             id = common_sampler_shift(smpl, ctx, -1, id);
         }
 
-        for (auto l_b : params.sparams.logit_bias) {
-            if (l_b.bias < -99 && id == l_b.token) {
-                std::string c_bias_tkn_string = common_token_to_piece(ctx, id);
-                writeTextFile("logit_biasing.txt", std::format("Restricted: '{}';", c_bias_tkn_string));
+        int checks = 0;
+        while (checks < params.sparams.logit_bias_tokens_manual.size()) {
+            for (auto tkn : params.sparams.logit_bias_tokens_manual) {
+                ++checks;
+                if (id == tkn) {
+                    std::string c_bias_tkn_string = common_token_to_piece(ctx, id);
+                    writeTextFile("logit_biasing.txt", std::format("{}: Restricted: '{}';", params.sparams.seed, c_bias_tkn_string));
 
-                id = common_sampler_shift(smpl, ctx, -1, id);
+                    id = common_sampler_shift(smpl, ctx, -1, id);
 
-                c_bias_tkn_string = common_token_to_piece(ctx, id);
-                writeTextFile("logit_biasing.txt", std::format(" replaced with: '{}'\n", c_bias_tkn_string));
+                    c_bias_tkn_string = common_token_to_piece(ctx, id);
+                    writeTextFile("logit_biasing.txt", std::format(" replaced with: '{}'\n", c_bias_tkn_string));
+                }
             }
         }
 
@@ -2009,8 +2100,6 @@ class chat
 
         if (debug) printf("Starting initial prompt processing...\n");
 
-        get_logit_bias_str();
-
 
         std::string result;
         //std::cout << " * " << std::endl;
@@ -2075,9 +2164,9 @@ class chat
     const std::string getTknFromEmbd(){
         if (debug) printf("-gp");
 
-        for (auto id : embd) { 
-            //return llama_token_to_string(ctx, id); 
-            return common_token_to_piece(ctx, id); 
+        for (auto id : embd) {
+            //return llama_token_to_string(ctx, id);
+            return common_token_to_piece(ctx, id);
         }
     }
 
@@ -2224,14 +2313,36 @@ class chat
         return getTknFromEmbd();
     }
 
+    std::string getMultiBit(int numTkns = 2, bool emptyMessage = false, bool shortMessage = false) { // 1 2 3 4
+        std::string result = "";
+
+        for (int i = 0; i < numTkns; i++) {
+            if (checkAndClearEmbd() == 0) {
+                finished = true;
+                return txt_vocab_eos;
+            }
+
+            if (!is_interacting) sampleTknIntoEmbd(emptyMessage, shortMessage); // 2
+
+            result += getTknFromEmbd();
+
+            if (llama_token_is_eog(vocab, common_sampler_last(smpl))) {
+                return result;
+            }
+        }
+
+        return result;
+    }
+
     // token by token generation and pushing
     std::string cycleStringsOnly(bool emptyMessage = false, bool shortMessage = false) {
 
         dynamicParamsPrepare();
         //process_prompt(false);  // do not forget to include it elsewhere after loading the model  
         //inputOnly(input); // MOVED
 
-        std::string bit = getBit(emptyMessage, shortMessage);
+        // std::string bit = getBit(emptyMessage, shortMessage);
+        std::string bit = getMultiBit(2, emptyMessage, shortMessage);
 
         if ((int) std::size(embd_inp) <= n_consumed) {
             if (debug) printf("-cso");

base_sampling2/common.h

@@ -200,6 +200,9 @@ struct common_params_sampling {
     std::vector<std::string> logit_bias_strings_beginning; // words for logit biases, beginning of the word matches
     std::vector<std::string> logit_bias_strings_ending; // words for logit biases, ending of the word matches
 
+    std::vector<llama_token> logit_bias_tokens_manual; // tokens for manual restricting
+    std::vector<std::string> logit_bias_strings_manual; // words for manual restricting
+
 
     std::map<std::string, float> logit_bias_strings_ext; // words for logit biases, but with extra configuration
     std::vector<std::string> logit_bias_strings_start; // restricted beginnings of messages

base_sampling2/include/jsonParams.h

@@ -551,6 +551,7 @@ static void getSamplingParamsFromJson(nlohmann::json& config, common_params& par
     if (checkJNum(config, "dry_base")) params.sparams.dry_base = config["dry_base"];
     if (checkJNum(config, "dry_allowed_length")) params.sparams.dry_allowed_length = config["dry_allowed_length"];
     if (checkJNum(config, "dry_penalty_last_n")) params.sparams.dry_penalty_last_n = config["dry_penalty_last_n"];
+    if (checkJArr(config, "dry_sequence_breakers")) params.sparams.dry_sequence_breakers = config["dry_sequence_breakers"];
 
 //mirostat
     if (checkJNum(config, "mirostat")) params.sparams.mirostat = config["mirostat"];
@@ -562,6 +563,7 @@ static void getSamplingParamsFromJson(nlohmann::json& config, common_params& par
     if (checkJArr(config, "logit_bias_strings_exact")) params.sparams.logit_bias_strings_exact = config["logit_bias_strings_exact"];
     if (checkJArr(config, "logit_bias_strings_beginning")) params.sparams.logit_bias_strings_beginning = config["logit_bias_strings_beginning"];
     if (checkJArr(config, "logit_bias_strings_ending")) params.sparams.logit_bias_strings_ending = config["logit_bias_strings_ending"];
+    if (checkJArr(config, "logit_bias_strings_manual")) params.sparams.logit_bias_strings_manual = config["logit_bias_strings_manual"];
 
 
     if (checkJObj(config, "logit_bias_strings_ext")) params.sparams.logit_bias_strings_ext = config["logit_bias_strings_ext"];

base_sampling2/master/ggml/src/ggml-cpu/ggml-cpu-impl.h

@@ -518,11 +518,14 @@ void ggml_barrier(struct ggml_threadpool * tp);
 #elif defined(__GNUC__)
 // GCC/Clang on *nix
 # define GGML_WEAK_ALIAS(name, alias) GGML_DO_PRAGMA(weak name = alias) // NOLINT
-#elif defined(_MSC_VER) && defined (_WIN64)
+#elif defined(_MSC_VER) && defined(_WIN64)
 // MSVC
 // Note: C name mangling varies across different calling conventions
 // see https://learn.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170
 # define GGML_WEAK_ALIAS(name, alias) GGML_DO_PRAGMA(comment(linker, "/alternatename:" #name "=" #alias))
+#elif defined(_MSC_VER) && defined(WIN32)
+// ref: https://github.com/ggml-org/whisper.cpp/pull/3239#issuecomment-2958224591
+# define GGML_WEAK_ALIAS(name, alias) GGML_DO_PRAGMA(comment(linker, "/alternatename:_" #name "=_" #alias))
 #else
 # error "Unsupported compiler for GGML_WEAK_ALIAS"
 #endif

base_sampling2/master/ggml/src/ggml-opencl/CMakeLists.txt

@@ -80,6 +80,7 @@ set(GGML_OPENCL_KERNELS
     mul_mv_q4_0_f32_1d_8x_flat
     mul_mv_q4_0_f32_1d_16x_flat
     mul_mv_q6_k
+    mul_mv_id_q4_0_f32_8x_flat
     mul
     norm
     relu