Merge pull request #170 from kozistr/fix/ranger21-wd

[Fix] weight decay in Ranger21 optimizer

Author: kozistr

docs/changelogs/v2.9.1.md

@@ -0,0 +1,9 @@
+## Change Log
+
+### Fix
+
+* fix weight decay in Ranger21 (#170)
+
+## Diff
+
+[2.9.0...2.9.1](https://github.com/kozistr/pytorch_optimizer/compare/v2.9.0...v2.9.1)

poetry.lock

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "pytorch_optimizer"
-version = "2.9.0"
-description = "optimizer & lr scheduler implementations in PyTorch with clean-code, strict types. Also, including useful optimization ideas."
+version = "2.9.1"
+description = "optimizer & lr scheduler collections in PyTorch"
 license = "Apache-2.0"
 authors = ["kozistr <[email protected]>"]
 maintainers = ["kozistr <[email protected]>"]

pyproject.toml

@@ -6,7 +6,7 @@
 def agc(p: torch.Tensor, grad: torch.Tensor, agc_eps: float, agc_clip_val: float, eps: float = 1e-6) -> torch.Tensor:
     r"""Clip gradient values in excess of the unit wise norm.
 
-    :param p: Parameter. parameter.
+    :param p: torch.Tensor. parameter.
     :param grad: torch.Tensor, gradient.
     :param agc_eps: float. agc epsilon to clip the norm of parameter.
     :param agc_clip_val: float. norm clip.

pytorch_optimizer/optimizer/agc.py

@@ -251,7 +251,7 @@ def step(self, closure: CLOSURE = None) -> LOSS:
                 self.apply_weight_decay(
                     p=p,
                     grad=None,
-                    lr=group['lr'],
+                    lr=lr,
                     weight_decay=group['weight_decay'],
                     weight_decouple=group['weight_decouple'],
                     fixed_decay=group['fixed_decay'],

pytorch_optimizer/optimizer/ranger21.py

@@ -19,15 +19,15 @@ numpy==1.21.1 ; python_full_version >= "3.7.2" and python_version < "3.8"
 numpy==1.24.3 ; python_version >= "3.8" and python_full_version < "4.0.0"
 packaging==23.1 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"
 pathspec==0.11.1 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"
-platformdirs==3.5.0 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"
+platformdirs==3.5.1 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"
 pluggy==1.0.0 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"
 pytest-cov==4.0.0 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"
 pytest==7.3.1 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"
 ruff==0.0.264 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"
-sympy==1.11.1 ; python_version >= "3.8" and python_full_version < "4.0.0"
+sympy==1.12 ; python_version >= "3.8" and python_full_version < "4.0.0"
 tomli==2.0.1 ; python_full_version >= "3.7.2" and python_full_version <= "3.11.0a6"
 torch==1.13.1+cpu ; python_full_version >= "3.7.2" and python_version < "3.8"
-torch==2.0.0+cpu ; python_version >= "3.8" and python_full_version < "4.0.0"
+torch==2.0.1+cpu ; python_version >= "3.8" and python_full_version < "4.0.0"
 typed-ast==1.5.4 ; python_version < "3.8" and implementation_name == "cpython" and python_full_version >= "3.7.2"
 typing-extensions==4.5.0 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"
 zipp==3.15.0 ; python_full_version >= "3.7.2" and python_version < "3.8"

requirements-dev.txt

@@ -7,7 +7,7 @@ mpmath==1.3.0 ; python_version >= "3.8" and python_full_version < "4.0.0"
 networkx==3.1 ; python_version >= "3.8" and python_full_version < "4.0.0"
 numpy==1.21.1 ; python_full_version >= "3.7.2" and python_version < "3.8"
 numpy==1.24.3 ; python_version >= "3.8" and python_full_version < "4.0.0"
-sympy==1.11.1 ; python_version >= "3.8" and python_full_version < "4.0.0"
+sympy==1.12 ; python_version >= "3.8" and python_full_version < "4.0.0"
 torch==1.13.1+cpu ; python_full_version >= "3.7.2" and python_version < "3.8"
-torch==2.0.0+cpu ; python_version >= "3.8" and python_full_version < "4.0.0"
+torch==2.0.1+cpu ; python_version >= "3.8" and python_full_version < "4.0.0"
 typing-extensions==4.5.0 ; python_full_version >= "3.7.2" and python_full_version < "4.0.0"