llama_save_model_to_file

Author: JohannesGaessler

common/common.cpp

@@ -1959,3 +1959,19 @@ common_control_vector_data common_control_vector_load(const std::vector<common_c
     return result;
 }
 
+ggml_opt_dataset_t common_opt_dataset_init(struct llama_context * ctx, const std::vector<llama_token> & tokens, int64_t stride) {
+    const int64_t ne_datapoint = llama_n_ctx(ctx);
+    const int64_t ndata        = (tokens.size() - ne_datapoint - 1) / stride;
+    ggml_opt_dataset_t result = ggml_opt_dataset_init(
+        GGML_TYPE_I32, GGML_TYPE_I32, ne_datapoint, ne_datapoint, ndata, /*ndata_shard =*/ 1);
+
+    llama_token * data   = (llama_token *) ggml_opt_dataset_data(result)->data;
+    llama_token * labels = (llama_token *) ggml_opt_dataset_labels(result)->data;
+
+    for (int64_t idata = 0; idata < ndata; ++idata) {
+        memcpy(data   + idata*ne_datapoint, tokens.data() + idata*stride + 0, ne_datapoint*sizeof(llama_token));
+        memcpy(labels + idata*ne_datapoint, tokens.data() + idata*stride + 1, ne_datapoint*sizeof(llama_token));
+    }
+
+    return result;
+}

common/common.h

@@ -610,3 +610,9 @@ common_control_vector_data common_control_vector_load(const std::vector<common_c
 static const char * const LLM_KV_SPLIT_NO            = "split.no";
 static const char * const LLM_KV_SPLIT_COUNT         = "split.count";
 static const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";
+
+//
+// training utils
+//
+
+ggml_opt_dataset_t common_opt_dataset_init(struct llama_context * ctx, const std::vector<llama_token> & tokens, int64_t stride);

examples/training/finetune.cpp

@@ -79,24 +79,35 @@ int main(int argc, char ** argv) {
     constexpr float val_split = 0.05f;
 
     std::vector<llama_token> tokens = common_tokenize(ctx, params.prompt, true);
-    ggml_opt_dataset_t dataset = llama_opt_dataset_init(ctx, tokens.data(), tokens.size(), llama_n_ctx(ctx)/2);
-    llama_opt_init(ctx);
+    ggml_opt_dataset_t dataset = common_opt_dataset_init(ctx, tokens, llama_n_ctx(ctx)/2);
+
+    struct ggml_opt_optimizer_params optimizer_params = ggml_opt_get_default_optimizer_params(nullptr);
+    optimizer_params.adamw.alpha = 1e-6f; // learning rate
+
+    struct llama_opt_params lopt_params {
+        /*n_ctx_train     =*/ 0,
+        /*get_opt_pars    =*/ ggml_opt_get_constant_optimizer_params,
+        /*get_opt_pars_ud =*/ &optimizer_params,
+    };
+    llama_opt_init(ctx, model, lopt_params);
+
     const int64_t idata_split = ggml_opt_dataset_ndata(dataset) * (1.0f - val_split);
 
-    while (true) {
-        ggml_opt_result_t result_train = ggml_opt_result_init();
-        ggml_opt_result_t result_eval  = ggml_opt_result_init();
+    ggml_opt_result_t result_train = ggml_opt_result_init();
+    ggml_opt_result_t result_eval  = ggml_opt_result_init();
 
+    for (int epoch = 0; epoch < 1; ++epoch) {
         llama_opt_epoch(ctx, dataset, result_train, result_eval, idata_split,
             ggml_opt_epoch_callback_progress_bar, ggml_opt_epoch_callback_progress_bar);
         fprintf(stderr, "\n");
 
-        ggml_opt_result_free(result_train);
-        ggml_opt_result_free(result_eval);
+        ggml_opt_result_reset(result_train);
+        ggml_opt_result_reset(result_eval);
     }
+    ggml_opt_result_free(result_train);
+    ggml_opt_result_free(result_eval);
 
-    LOG("\n");
-    llama_perf_context_print(ctx);
+    llama_save_model_to_file(model, "finetuned-model.gguf");
 
     llama_free(ctx);
     llama_free_model(model);

ggml/include/ggml-opt.h

@@ -90,10 +90,13 @@ extern "C" {
     // userdata can be used to pass arbitrary data
     typedef struct ggml_opt_optimizer_params (*ggml_opt_get_optimizer_params)(void * userdata);
 
-    // returns the default optimizer params (constant)
+    // returns the default optimizer params (constant, hard-coded values)
     // userdata is not used
     GGML_API struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata);
 
+    // casts userdata to ggml_opt_optimizer_params and returns it
+    GGML_API struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata);
+
     // parameters for initializing a new optimization context
     struct ggml_opt_params {
         ggml_backend_sched_t backend_sched; // defines which backends are used to construct the compute graphs

ggml/src/ggml-opt.cpp

@@ -229,6 +229,10 @@ struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * us
     return result;
 }
 
+struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata) {
+    return *((struct ggml_opt_optimizer_params *) userdata);
+}
+
 struct ggml_opt_params ggml_opt_default_params(
         ggml_backend_sched_t      backend_sched,
         struct ggml_context     * ctx_compute,

include/llama.h

@@ -413,6 +413,10 @@ extern "C" {
                              const char * path_model,
               struct llama_model_params   params);
 
+    LLAMA_API void llama_save_model_to_file(
+            const struct llama_model * model,
+                        const char * path_model);
+
     LLAMA_API void llama_free_model(struct llama_model * model);
 
     // TODO: rename to llama_init_from_model
@@ -1255,9 +1259,14 @@ extern "C" {
     // training
     //
 
-    LLAMA_API ggml_opt_dataset_t llama_opt_dataset_init(struct llama_context * ctx, const llama_token * tokens, int64_t n_tokens, int32_t stride);
+    struct llama_opt_params {
+        uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0
+
+        ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters
+        void * get_opt_pars_ud;                     // userdata for calculating optimizer parameters
+    };
 
-    LLAMA_API void llama_opt_init(struct llama_context * lctx);
+    LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params);
 
     LLAMA_API void llama_opt_epoch(
             struct llama_context    * lctx,

src/llama-vocab.h

@@ -61,6 +61,9 @@ struct llama_vocab {
     // set of all tokens that cause "end of generation"
     std::set<id> special_eog_ids;
 
+    std::string tokenizer_model;
+    std::string tokenizer_pre;
+
     // tokenizer flags
     bool tokenizer_add_space_prefix           = false;
     bool tokenizer_add_bos                    = false;