class LlamaSampler: append add_dry()

Fix the char array params convert problem

Author: JamePeng

llama_cpp/_internals.py

@@ -821,20 +821,51 @@ def add_grammar(self, model: LlamaModel, grammar: LlamaGrammar):
         )
         self._add_sampler(sampler)
 
+    def convert_list_str_to_char_ptr_array(str_list: List[str]) -> ctypes.POINTER(ctypes.POINTER(ctypes.c_char)):
+        """
+        Converts a list of strings to a char** array for C interop.
+        Args:
+            list[str]: List of string objects.
+        Returns:
+            A ctypes pointer to a char** array.
+        """
+        # Encode strings to bytes
+        byte_list = [s.encode('utf-8') for s in str_list]
+        # Calculate the number of breakers
+        num_byte_list= len(byte_list)
+        # Define the type of a char pointer
+        char_ptr_type = ctypes.POINTER(ctypes.c_char)
+        # Define the type of an array of char pointers
+        char_ptr_array_type = char_ptr_type * num_byte_list
+
+        # Allocate memory for the array of char pointers
+        char_ptr_array = char_ptr_array_type()
+
+        # Populate the array with pointers to the byte strings
+        for i, byte_string in enumerate(byte_list):
+            # Create a null-terminated C-style string buffer
+            c_char_array = ctypes.create_string_buffer(byte_string)
+            # Cast the buffer to a char pointer and assign it to the array
+            char_ptr_array[i] = ctypes.cast(c_char_array, char_ptr_type)
+
+        # Cast the array to a char** pointer and return it
+        return ctypes.cast(char_ptr_array, ctypes.POINTER(char_ptr_type)), num_byte_list
+
     def add_grammar_lazy(
             self,
             model: LlamaModel,
             grammar: LlamaGrammar,
-            trigger_words: list[bytes],
-            num_trigger_words: int,
             trigger_tokens:list[llama_cpp.llama_token],
-            num_trigger_tokens: int
+            num_trigger_tokens: int,
+            trigger_words: list[str]=[]
         ):
+        trigger_words_char_ptr_array, num_trigger_words = self.convert_list_str_to_char_ptr_array(trigger_words)
+
         sampler = llama_cpp.llama_sampler_init_grammar_lazy(
             model.vocab,
             grammar._grammar.encode("utf-8"),
             grammar._root.encode("utf-8"),
-            trigger_words,
+            trigger_words_char_ptr_array,
             num_trigger_words,
             trigger_tokens,
             num_trigger_tokens
@@ -845,16 +876,17 @@ def add_grammar_lazy_patterns(
             self,
             model: LlamaModel,
             grammar: LlamaGrammar,
-            trigger_patterns: list[bytes],
             num_trigger_patterns: int,
             trigger_tokens:list[llama_cpp.llama_token],
-            num_trigger_tokens: int
+            num_trigger_tokens: int,
+            trigger_patterns: list[str]=[]
         ):
+        trigger_patterns_char_ptr_array, num_trigger_patterns = self.convert_list_str_to_char_ptr_array(trigger_patterns)
         sampler = llama_cpp.llama_sampler_init_grammar_lazy_patterns(
             model.vocab,
             grammar._grammar.encode("utf-8"),
             grammar._root.encode("utf-8"),
-            trigger_patterns,
+            trigger_patterns_char_ptr_array,
             num_trigger_patterns,
             trigger_tokens,
             num_trigger_tokens
@@ -882,6 +914,29 @@ def add_penalties(
         )
         self._add_sampler(sampler)
 
+    def add_dry(
+        self,
+        model: LlamaModel,
+        n_ctx_train: int,
+        dry_multiplier: float,
+        dry_base: float,
+        dry_allowed_length: int,
+        dry_penalty_last_n: int,
+        seq_breakers: list[str] = []
+    ):
+        seq_breakers_bytes_char_ptr_array, num_breakers = self.convert_list_str_to_char_ptr_array(seq_breakers)
+        sampler = llama_cpp.llama_sampler_init_dry(
+            model.vocab,
+            n_ctx_train,
+            dry_multiplier,
+            dry_base,
+            dry_allowed_length,
+            dry_penalty_last_n,
+            seq_breakers_bytes_char_ptr_array,
+            num_breakers
+        )
+        self._add_sampler(sampler)
+
     def init_logit_bias(
         self, n_vocab: int, n_logit_bias, logit_bias: llama_cpp.llama_logit_bias_p
     ):

llama_cpp/llama_cpp.py

@@ -3413,7 +3413,7 @@ class llama_sampler(ctypes.Structure):
     ("clone", llama_sampler_i_clone),
     ("free", llama_sampler_i_free),
 ]
-llama_sampler_i_p = CtypesPointer[llama_sampler_i]
+llama_sampler_i_p = ctypes.POINTER(llama_sampler_i)
 
 # // mirror of llama_sampler_i: