SGD op param store weight-decay and not 1-alpha*wd

Author: graehl

ggml/src/ggml-cpu/ops.cpp

@@ -10387,7 +10387,7 @@ void ggml_compute_forward_opt_step_adamw(
 static void ggml_compute_forward_opt_step_sgd_f32(const ggml_compute_params * params, ggml_tensor * dst) {
     const ggml_tensor * src0         = dst->src[0];
     const ggml_tensor * src0_grad    = dst->src[1];
-    const ggml_tensor * sgd_params = dst->src[2];
+    const ggml_tensor * sgd_params   = dst->src[2];
 
     GGML_ASSERT(ggml_are_same_shape(src0, src0_grad));
     GGML_ASSERT(ggml_nelements(sgd_params) == 2);
@@ -10410,7 +10410,7 @@ static void ggml_compute_forward_opt_step_sgd_f32(const ggml_compute_params * pa
     // using adamw param subset we care about - alpha, wd - could have a separate struct
     const float * sgd_params_ptr   = ggml_get_data_f32(sgd_params);
     const float   alpha            = sgd_params_ptr[0];
-    const float   keep             = sgd_params_ptr[1];
+    const float   keep             = 1.f - alpha * sgd_params_ptr[1];
 
     for (int ir = ir0; ir < ir1; ++ir) {
         const int64_t i03 = ir / (ne02 * ne01);

ggml/src/ggml-cuda/opt-step-sgd.cu

@@ -11,7 +11,7 @@ static __global__ void opt_step_sgd_f32(
 
     if (i >= k)
         return;
-    x[i] = x[i] * pars[1] - pars[0] * g[i];
+    x[i] = x[i] * (1.f - pars[0] * pars[1]) - pars[0] * g[i];
 }
 
 static void opt_step_sgd_f32_cuda(

ggml/src/ggml-opt.cpp

@@ -824,7 +824,8 @@ void ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result) {
                     GGML_ASSERT(opt_pars.sgd.wd >= 0.0f);
                     GGML_ASSERT(opt_pars.sgd.wd <= 1.0f);
                     float * sgd = ggml_get_data_f32(opt_ctx->adamw_params);
-                    sgd[1]      = 1. - (sgd[0] = opt_pars.sgd.alpha) * opt_pars.sgd.wd;
+                    sgd[0] = opt_pars.sgd.alpha;
+                    sgd[1] = opt_pars.sgd.wd;
                 }
                 break;