We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 23f32fa commit acd218bCopy full SHA for acd218b
pytorch_optimizer/optimizer/adam_mini.py
@@ -246,16 +246,18 @@ def step_lefts(
246
if state['reduced']:
247
dist.all_reduce(tmp_lr, op=dist.ReduceOp.SUM)
248
249
- tmp_lr.div_(state['dim'])
+ tmp_lr.div_(state['dimension'])
250
251
m, v = state['m'], state['v_mean']
252
253
m.lerp_(grad, weight=1.0 - beta1)
254
- v.mul_(beta2).add_(tmp_lr, value=1.0 - beta2)
+ v.mul_(beta2).add_(tmp_lr, alpha=1.0 - beta2)
255
256
h = (v.sqrt() / bias_correction2_sq).add_(eps)
257
258
- update = 1 / (bias_correction1 * h).mul_(m)
+ stepsize = (1 / bias_correction1) / h
259
+
260
+ update = m * stepsize
261
262
p.add_(update, alpha=-lr)
263
0 commit comments