From 39a6d3b5535b05785b2ca3ea5177c37db5c5ca76 Mon Sep 17 00:00:00 2001
From: biplavbarua <innocentsavage.bb@gmail.com>
Date: Sat, 3 Jan 2026 13:38:43 +0530
Subject: [PATCH] feat: add SGD/Adam optimizers and tanh/sigmoid activations

---
 .gitignore          |  6 ++++++
 micrograd/engine.py | 24 +++++++++++++++++++++-
 micrograd/optim.py  | 50 +++++++++++++++++++++++++++++++++++++++++++++
 test/test_optim.py  | 41 +++++++++++++++++++++++++++++++++++++
 4 files changed, 120 insertions(+), 1 deletion(-)
 create mode 100644 micrograd/optim.py
 create mode 100644 test/test_optim.py

diff --git a/.gitignore b/.gitignore
index 87620ac7..59812ea8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,7 @@
 .ipynb_checkpoints/
+__pycache__/
+*.py[cod]
+*$py.class
+venv/
+.env
+.DS_Store
diff --git a/micrograd/engine.py b/micrograd/engine.py
index afd82cc5..d2a54030 100644
--- a/micrograd/engine.py
+++ b/micrograd/engine.py
@@ -1,4 +1,4 @@
-
+import math
 class Value:
     """ stores a single scalar value and its gradient """
 
@@ -42,6 +42,28 @@ def _backward():
 
         return out
 
+    def tanh(self):
+        x = self.data
+        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)
+        out = Value(t, (self,), 'tanh')
+
+        def _backward():
+            self.grad += (1 - t**2) * out.grad
+        out._backward = _backward
+
+        return out
+
+    def sigmoid(self):
+        x = self.data
+        s = 1 / (1 + math.exp(-x))
+        out = Value(s, (self,), 'sigmoid')
+
+        def _backward():
+            self.grad += (s * (1 - s)) * out.grad
+        out._backward = _backward
+
+        return out
+
     def relu(self):
         out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')
 
diff --git a/micrograd/optim.py b/micrograd/optim.py
new file mode 100644
index 00000000..327d693a
--- /dev/null
+++ b/micrograd/optim.py
@@ -0,0 +1,50 @@
+from micrograd.engine import Value
+
+class Optimizer:
+    def __init__(self, parameters):
+        self.parameters = [p for p in parameters if p.grad != 0 or True] # keep all for now
+
+    def step(self):
+        raise NotImplementedError
+
+    def zero_grad(self):
+        for p in self.parameters:
+            p.grad = 0
+
+class SGD(Optimizer):
+    def __init__(self, parameters, lr=0.01, momentum=0.0):
+        super().__init__(parameters)
+        self.lr = lr
+        self.momentum = momentum
+        self.velocities = {p: 0 for p in self.parameters}
+
+    def step(self):
+        for p in self.parameters:
+            v = self.velocities[p]
+            v = self.momentum * v - self.lr * p.grad
+            self.velocities[p] = v
+            p.data += v
+
+class Adam(Optimizer):
+    def __init__(self, parameters, lr=0.001, betas=(0.9, 0.999), eps=1e-8):
+        super().__init__(parameters)
+        self.lr = lr
+        self.beta1, self.beta2 = betas
+        self.eps = eps
+        self.m = {p: 0 for p in self.parameters}
+        self.v = {p: 0 for p in self.parameters}
+        self.t = 0
+
+    def step(self):
+        self.t += 1
+        for p in self.parameters:
+            if p.grad == 0:
+                continue
+            
+            self.m[p] = self.beta1 * self.m[p] + (1 - self.beta1) * p.grad
+            self.v[p] = self.beta2 * self.v[p] + (1 - self.beta2) * (p.grad ** 2)
+            
+            m_hat = self.m[p] / (1 - self.beta1 ** self.t)
+            v_hat = self.v[p] / (1 - self.beta2 ** self.t)
+            
+            p.data -= self.lr * m_hat / (v_hat**0.5 + self.eps)
diff --git a/test/test_optim.py b/test/test_optim.py
new file mode 100644
index 00000000..1add1924
--- /dev/null
+++ b/test/test_optim.py
@@ -0,0 +1,41 @@
+import unittest
+import torch
+from micrograd.engine import Value
+from micrograd.optim import SGD, Adam
+
+class TestOptim(unittest.TestCase):
+
+    def test_sgd(self):
+        # Simple quadratic optimization: y = (x - 3)^2
+        # Minimum at x = 3
+        
+        # micrograd
+        x = Value(0.0)
+        optimizer = SGD([x], lr=0.1)
+        
+        for _ in range(100):
+            optimizer.zero_grad()
+            loss = (x - 3)**2
+            loss.backward()
+            optimizer.step()
+            
+        self.assertAlmostEqual(x.data, 3.0, delta=0.1)
+
+    def test_adam(self):
+        # Simple quadratic optimization: y = (x - 3)^2
+        # Minimum at x = 3
+        
+        # micrograd
+        x = Value(0.0)
+        optimizer = Adam([x], lr=0.1)
+        
+        for _ in range(300):
+            optimizer.zero_grad()
+            loss = (x - 3)**2
+            loss.backward()
+            optimizer.step()
+            
+        self.assertAlmostEqual(x.data, 3.0, delta=0.01)
+
+if __name__ == '__main__':
+    unittest.main()