Skip to content

Commit b73f618

Browse files
graehlgraehl
committed
examples/finetune -opt SGD (stochastic gradient descent) memory opt
add unit tested GGML_OPT_OPTIMIZER_SGD to ggml - avoids allocating m, v tensors. support finetune.cpp arg -opt SGD (or sgd). (default adamw as before) llama 3.2-1b-F32 result: observed 11gb gpu ram (41 sec/epoch) when using SGD instead of 19gb (55 sec/epoch) using adamw. (wikipedia 100 lines finetune) ( using the same GPU memory, adamw can only do before OOM 512 batch/context, reaching: train: [███████▉] data=0000140/0000140 loss=0.02575±0.00099 acc=99.52±0.03% t=00:00:47 ETA=00:00:00 val: [███████▉] data=0000008/0000008 loss=4.76565±0.28810 acc=41.46±0.77% t=00:00:00 ETA=00:00:00 SGD is superior, though it converges slower, with max before OOM 1728 batch/context (esp see the better validation perf): train: [███████▉] data=0000039/0000039 loss=0.00371±0.00010 acc=99.96±0.01% t=00:00:41 ETA=00:00:00 val: [███████▉] data=0000003/0000003 loss=5.11406±0.76034 acc=48.01±0.69% t=00:00:01 ETA=00:00:00 ) note: when finetuning long enough (or w/ enough -lr), validation accuracy *eventually* drops ('catastrophic forgetting') -lr-half (halflife) option useful for SGD to avoid oscillation or super slow underdamped learning (makes setting -lr more forgiving). terminal -lr for now is set by lr-halvings i.e. if you want at most 1/8 the inital -lr you set -lr-halvings 3. note: objective loss not directly comparable between adamw, sgd? - check perplexity or accuracy or consider relative improvements for convergence new finetune args -wd 1e-9 to enable weight decay in sgd or adamw, and max -epochs N (default 2 as before) cache (1 - wd*alpha) in 'adamw' opt struct - no noticeable perf benefit, disabled (still done for new SGD though) since opt. memory is pre-allocated, the ggml_opt_get_optimizer_params would probably be able to change between SGD and AdamW with each epoch but would need to use adamw for the first (unconfirmed - no cmdline arg to set such a policy yet) test-opt checks adamw as before and now sgd (except for a few disabled tests for sgd only; probably just needs logging values and adding alternate reference values); tolerance on the 'regression' test is broader for sgd (so we don't need many more epochs)
1 parent 55c2646 commit b73f618

File tree

23 files changed

+729
-213
lines changed

23 files changed

+729
-213
lines changed

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
1212
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
1313
endif()
1414

15+
message("CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
16+
1517
# Add path to modules
1618
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
1719

‎common/arg.cpp‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1196,6 +1196,7 @@ bool common_params_parse(int argc, char ** argv, common_params & params, llama_e
11961196
common_params_print_completion(ctx_arg);
11971197
exit(0);
11981198
}
1199+
params.lr.init();
11991200
} catch (const std::invalid_argument & ex) {
12001201
fprintf(stderr, "%s\n", ex.what());
12011202
ctx_arg.params = params_org;
@@ -2609,9 +2610,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
26092610
{"-o", "--output", "--output-file"}, "FNAME",
26102611
string_format("output file (default: '%s')", params.out_file.c_str()),
26112612
[](common_params & params, const std::string & value) {
2612-
params.out_file = value;
2613+
params.out_file = value;
26132614
}
2614-
).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_TTS}));
2615+
).set_examples({LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_CVECTOR_GENERATOR, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_FINETUNE}));
26152616
add_opt(common_arg(
26162617
{"-ofreq", "--output-frequency"}, "N",
26172618
string_format("output the imatrix every N iterations (default: %d)", params.n_out_freq),
@@ -3416,5 +3417,51 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
34163417
}
34173418
).set_examples({LLAMA_EXAMPLE_SERVER}));
34183419

3420+
add_opt(
3421+
common_arg({ "-lr", "--learning-rate-initial" }, "ALPHA",
3422+
string_format(
3423+
"adamw or sgd optimizer alpha (default: %.2g); note: sgd alpha recommended ~10x (no momentum)",
3424+
(double) params.lr.lr0),
3425+
[](common_params & params, const std::string & value) { params.lr.lr0 = std::stof(value); })
3426+
.set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3427+
add_opt(
3428+
common_arg({ "-lr-min", "--learning-rate-min" }, "ALPHA",
3429+
string_format(
3430+
"(if >0) final learning rate (default=%.2g)",
3431+
(double) params.lr.lr_min),
3432+
[](common_params & params, const std::string & value) { params.lr.lr_min = std::stof(value); })
3433+
.set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3434+
add_opt(
3435+
common_arg({ "-min-epochs", "--learning-rate-min-epochs" }, "ALPHA",
3436+
string_format(
3437+
"(if >0) reach -lr-min after this many epochs (instead of only at the last) (default=%.2g)",
3438+
(double) params.lr.min_epochs),
3439+
[](common_params & params, const std::string & value) { params.lr.min_epochs = std::stof(value); })
3440+
.set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3441+
add_opt(common_arg(
3442+
{ "-wd", "--weight-decay" }, "WD",
3443+
string_format(
3444+
"adamw or sgd optimizer weight decay (0 is off; recommend very small e.g. 1e-9) (default: %.2g).",
3445+
(double) params.lr.wd),
3446+
[](common_params & params, const std::string & value) { params.lr.wd = std::stof(value); })
3447+
.set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3448+
add_opt(common_arg({ "-val", "--val-split" }, "FRACTION",
3449+
string_format("fraction of data to use as validation set for training (default: %.2g).",
3450+
(double) params.val_split),
3451+
[](common_params & params, const std::string & value) { params.val_split = std::stof(value); })
3452+
.set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3453+
add_opt(common_arg({ "-epochs", "--epochs" }, "N",
3454+
string_format("optimizer max # of epochs (default: %d)", params.lr.epochs),
3455+
[](common_params & params, int epochs) { params.lr.epochs = epochs; })
3456+
.set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3457+
add_opt(common_arg({ "-opt", "--optimizer" }, "sgd|adamw", "adamw or sgd",
3458+
[](common_params & params, const std::string & name) {
3459+
params.optimizer = common_opt_get_optimizer(name.c_str());
3460+
if (params.optimizer == GGML_OPT_OPTIMIZER_TYPE_COUNT) {
3461+
throw std::invalid_argument("invalid --optimizer, valid options: adamw, sgd");
3462+
}
3463+
})
3464+
.set_examples({ LLAMA_EXAMPLE_FINETUNE }));
3465+
34193466
return ctx_arg;
34203467
}

‎common/common.cpp‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
#endif
4242
#include <locale>
4343
#include <windows.h>
44+
#include <string.h>
4445
#include <fcntl.h>
4546
#include <io.h>
4647
#else
@@ -1548,3 +1549,53 @@ ggml_opt_dataset_t common_opt_dataset_init(struct llama_context * ctx, const std
15481549

15491550
return result;
15501551
}
1552+
1553+
ggml_opt_optimizer_params common_opt_lr_pars(void * userdata) {
1554+
ggml_opt_optimizer_params result = ggml_opt_get_default_optimizer_params(nullptr);
1555+
const lr_opt & d = *(lr_opt *) userdata;
1556+
result.adamw.alpha = result.sgd.alpha = d.get_lr(d.epoch);
1557+
result.sgd.wd = result.adamw.wd = d.wd;
1558+
return result;
1559+
}
1560+
1561+
static inline bool eq_case_insensitive(char const* a, char const* b) {
1562+
return !
1563+
#if defined(_MSC_VER)
1564+
_stricmp
1565+
#else
1566+
strcasecmp
1567+
#endif
1568+
(a, b);
1569+
}
1570+
1571+
GGML_API enum ggml_opt_optimizer_type common_opt_get_optimizer(const char * n) {
1572+
if (eq_case_insensitive("adamw", n)) {
1573+
return GGML_OPT_OPTIMIZER_TYPE_ADAMW;
1574+
} else if (eq_case_insensitive("sgd", n)) {
1575+
return GGML_OPT_OPTIMIZER_TYPE_SGD;
1576+
} else {
1577+
return GGML_OPT_OPTIMIZER_TYPE_COUNT;
1578+
}
1579+
}
1580+
1581+
static float const k_log_2 = std::log(2.f);
1582+
1583+
void lr_opt::init() {
1584+
if (lr_min > 0 && lr_min < lr0) {
1585+
float nhalf = std::log(lr0 / lr_min) / k_log_2;
1586+
float e = epochs;
1587+
if (min_epochs > 0 && min_epochs < e)
1588+
e = min_epochs;
1589+
else
1590+
min_epochs = e;
1591+
scale_epoch = nhalf / e;
1592+
}
1593+
}
1594+
1595+
float lr_opt::get_lr(float epoch) const {
1596+
float r = lr_min <= 0 ? lr0 :
1597+
epoch >= min_epochs ? lr_min :
1598+
lr0 * std::pow(.5, epoch * scale_epoch);
1599+
LOG_INF("epoch %.2g lr=%.2g\n", epoch, r);
1600+
return r;
1601+
}

‎common/common.h‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,17 @@
22

33
#pragma once
44

5-
#include "llama-cpp.h"
6-
75
#include <set>
6+
#include <sstream>
87
#include <string>
98
#include <string_view>
109
#include <vector>
1110
#include <map>
1211
#include <sstream>
12+
#include <cmath>
13+
14+
#include "ggml-opt.h"
15+
#include "llama-cpp.h"
1316

1417
#ifdef _WIN32
1518
#define DIRECTORY_SEPARATOR '\\'
@@ -81,6 +84,7 @@ enum llama_example {
8184
LLAMA_EXAMPLE_LOOKUP,
8285
LLAMA_EXAMPLE_PARALLEL,
8386
LLAMA_EXAMPLE_TTS,
87+
LLAMA_EXAMPLE_FINETUNE,
8488

8589
LLAMA_EXAMPLE_COUNT,
8690
};
@@ -223,6 +227,25 @@ enum common_reasoning_format {
223227
COMMON_REASONING_FORMAT_DEEPSEEK, // Extract thinking tag contents and return as `message.reasoning_content`, including in streaming deltas.
224228
};
225229

230+
231+
struct lr_opt {
232+
float lr0 = 1e-5; // learning rate at first epoch
233+
float lr_min = -1;
234+
float min_epochs = -1; // if >0, constant (lr_min) after this many epochs
235+
float scale_epoch = 0;
236+
float wd = 0;
237+
unsigned epochs = 2;
238+
239+
unsigned epoch; // set by optimizer outer (epochs) loop
240+
// learning rate decay - constant LR per epoch only for now
241+
float get_lr(float e) const;
242+
float get_lr() const { return get_lr(epoch); }
243+
// must call after arg parse, before get_lr
244+
void init();
245+
};
246+
247+
struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);
248+
226249
struct common_params {
227250
int32_t n_predict = -1; // new tokens to predict
228251
int32_t n_ctx = 4096; // context size
@@ -354,6 +377,12 @@ struct common_params {
354377
bool no_mmproj = false; // explicitly disable multimodal model
355378
std::vector<std::string> image; // path to image file(s)
356379

380+
// finetune
381+
struct lr_opt lr;
382+
enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;
383+
float val_split = 0.05f; // fraction of the data used for the validation set
384+
std::string opt_save_model_to = "finetuned-model.gguf";
385+
357386
// embedding
358387
bool embedding = false; // get only sentence embedding
359388
int32_t embd_normalize = 2; // normalisation for embeddings (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)
@@ -677,3 +706,6 @@ const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";
677706
//
678707

679708
ggml_opt_dataset_t common_opt_dataset_init(struct llama_context * ctx, const std::vector<llama_token> & tokens, int64_t stride);
709+
710+
// "adamw" or "sgd" (case insensitive)
711+
enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *);

‎examples/training/finetune.cpp‎

Lines changed: 36 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,31 @@
1-
#include "arg.h"
2-
#include "common.h"
3-
#include "log.h"
4-
#include "llama.h"
5-
61
#include <cmath>
72
#include <cstdio>
83
#include <cstring>
94
#include <ctime>
105
#include <vector>
116

7+
#include "arg.h"
8+
#include "common.h"
9+
#include "llama.h"
10+
#include "log.h"
11+
1212
#if defined(_MSC_VER)
13-
#pragma warning(disable: 4244 4267) // possible loss of data
13+
# pragma warning(disable : 4244 4267) // possible loss of data
1414
#endif
1515

16+
17+
1618
int main(int argc, char ** argv) {
1719
common_params params;
18-
1920
params.escape = false;
2021

21-
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_PERPLEXITY)) {
22+
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_FINETUNE)) {
2223
return 1;
2324
}
2425

2526
if (params.use_mmap) {
26-
LOG_INF("%s: force disabling memory mapping because it would result in-read-only pointers to the weights\n", __func__);
27+
LOG_INF("%s: force disabling memory mapping because it would result in-read-only pointers to the weights\n",
28+
__func__);
2729
params.use_mmap = false;
2830
}
2931
if (params.cache_type_k != GGML_TYPE_F32) {
@@ -38,11 +40,11 @@ int main(int argc, char ** argv) {
3840
common_init();
3941
llama_backend_init();
4042
llama_numa_init(params.numa);
41-
4243
// load the model and apply lora adapter, if any
43-
common_init_result llama_init = common_init_from_params(params);
44-
llama_model_ptr & model = llama_init.model;
45-
llama_context_ptr & ctx = llama_init.context;
44+
common_init_result llama_init = common_init_from_params(params);
45+
llama_model_ptr & model = llama_init.model;
46+
llama_context_ptr & ctx = llama_init.context;
47+
auto pctx = ctx.get();
4648

4749
if (model == NULL) {
4850
LOG_ERR("%s: unable to load model\n", __func__);
@@ -55,31 +57,32 @@ int main(int argc, char ** argv) {
5557
LOG_INF("%s\n", common_params_get_system_info(params).c_str());
5658
}
5759

58-
constexpr float val_split = 0.05f;
59-
60-
std::vector<llama_token> tokens = common_tokenize(ctx.get(), params.prompt, true);
61-
ggml_opt_dataset_t dataset = common_opt_dataset_init(ctx.get(), tokens, llama_n_ctx(ctx.get())/2);
62-
63-
struct ggml_opt_optimizer_params optimizer_params = ggml_opt_get_default_optimizer_params(nullptr);
64-
optimizer_params.adamw.alpha = 1e-7f; // learning rate
65-
66-
struct llama_opt_params lopt_params {
67-
/*n_ctx_train =*/ 0,
68-
/*param_filter =*/ llama_opt_param_filter_all,
69-
/*param_filter_ud =*/ nullptr,
70-
/*get_opt_pars =*/ ggml_opt_get_constant_optimizer_params,
71-
/*get_opt_pars_ud =*/ &optimizer_params,
60+
std::vector<llama_token> tokens = common_tokenize(pctx, params.prompt, true);
61+
ggml_opt_dataset_t dataset = common_opt_dataset_init(pctx, tokens, llama_n_ctx(pctx) / 2);
62+
63+
struct lr_opt & lr = params.lr;
64+
LOG_INF("-optimizer %s -lr0 %.2g -wd %.2g -lr-min %.2g -min-epochs %.2g -epochs %d -period %.2g -val %.2g\n",
65+
ggml_opt_optimizer_name(params.optimizer), (double) lr.lr0, (double) lr.wd, (double) lr.lr_min, (double) lr.min_epochs,
66+
(unsigned) lr.epochs, (double) params.n_batch / params.n_ubatch, (double) params.val_split);
67+
68+
struct llama_opt_params lopt_params{
69+
/*n_ctx_train =*/0,
70+
/*param_filter =*/llama_opt_param_filter_all,
71+
/*param_filter_ud =*/nullptr,
72+
/*get_opt_pars =*/common_opt_lr_pars,
73+
/*get_opt_pars_ud =*/&params.lr,
74+
/*optimizer_type =*/params.optimizer,
7275
};
73-
llama_opt_init(ctx.get(), model.get(), lopt_params);
76+
llama_opt_init(pctx, model.get(), lopt_params);
7477

75-
const int64_t idata_split = ggml_opt_dataset_ndata(dataset) * (1.0f - val_split);
78+
const int64_t idata_split = ggml_opt_dataset_ndata(dataset) * (1.0f - params.val_split);
7679

7780
ggml_opt_result_t result_train = ggml_opt_result_init();
7881
ggml_opt_result_t result_eval = ggml_opt_result_init();
7982

80-
for (int epoch = 0; epoch < 2; ++epoch) {
81-
llama_opt_epoch(ctx.get(), dataset, result_train, result_eval, idata_split,
82-
ggml_opt_epoch_callback_progress_bar, ggml_opt_epoch_callback_progress_bar);
83+
for (lr.epoch = 0; lr.epoch < lr.epochs; ++lr.epoch) {
84+
llama_opt_epoch(pctx, dataset, result_train, result_eval, idata_split,
85+
ggml_opt_epoch_callback_progress_bar, ggml_opt_epoch_callback_progress_bar);
8386
fprintf(stderr, "\n");
8487

8588
ggml_opt_result_reset(result_train);
@@ -88,7 +91,7 @@ int main(int argc, char ** argv) {
8891
ggml_opt_result_free(result_train);
8992
ggml_opt_result_free(result_eval);
9093

91-
llama_model_save_to_file(model.get(), "finetuned-model.gguf");
94+
llama_model_save_to_file(model.get(), params.out_file.c_str());
9295

9396
llama_backend_free();
9497

0 commit comments

Comments
 (0)