Implementation Guide

💻 Implementation Guide

Step-by-Step Code Walkthrough

---

🏗️ Architecture Overview

graph TB
    A[LogisticRegression Class] --> B[__init__]
    A --> C[fit]
    A --> D[predict]
    A --> E[predict_proba]
    A --> F[score]
    
    C --> C1[_sigmoid]
    C --> C2[_compute_cost]
    C --> C3[_compute_gradients]
    C --> C4[_update_parameters]
    
    style A fill:#e1f5ff
    style C fill:#ffe1e1
    style D fill:#e1ffe1
    style E fill:#fff4e1
    style F fill:#f0e1ff

Loading

---

📦 Class Structure

Complete Implementation

import numpy as np
import matplotlib.pyplot as plt

class LogisticRegression: 
    """
    Logistic Regression implementation from scratch
    
    Parameters:
    -----------
    learning_rate : float, default=0.01
        Step size for gradient descent
    n_iterations : int, default=1000
        Number of training iterations
    verbose : bool, default=False
        Print cost during training
    """
    
    def __init__(self, learning_rate=0.01, n_iterations=1000, verbose=False):
        self.learning_rate = learning_rate
        self. n_iterations = n_iterations
        self.verbose = verbose
        self.weights = None
        self.bias = None
        self.cost_history = []
    
    def _sigmoid(self, z):
        """Sigmoid activation function"""
        return 1 / (1 + np.exp(-np.clip(z, -500, 500)))
    
    def _compute_cost(self, y_true, y_pred):
        """Binary cross-entropy cost function"""
        m = len(y_true)
        epsilon = 1e-15
        y_pred = np.clip(y_pred, epsilon, 1 - epsilon)
        
        cost = -1/m * np. sum(
            y_true * np.log(y_pred) + 
            (1 - y_true) * np.log(1 - y_pred)
        )
        return cost
    
    def fit(self, X, y):
        """
        Train the logistic regression model
        
        Parameters:
        -----------
        X :  array-like, shape (m, n)
            Training features
        y : array-like, shape (m,)
            Training labels
        """
        m, n = X.shape
        
        # Initialize parameters
        self.weights = np. zeros(n)
        self.bias = 0
        
        # Gradient descent
        for i in range(self.n_iterations):
            # Forward propagation
            z = np.dot(X, self.weights) + self.bias
            predictions = self._sigmoid(z)
            
            # Compute cost
            cost = self._compute_cost(y, predictions)
            self.cost_history.append(cost)
            
            # Backward propagation
            dz = predictions - y
            dw = (1/m) * np.dot(X.T, dz)
            db = (1/m) * np.sum(dz)
            
            # Update parameters
            self.weights -= self. learning_rate * dw
            self.bias -= self.learning_rate * db
            
            # Print progress
            if self.verbose and i % 100 == 0:
                print(f"Iteration {i}:  Cost = {cost:.4f}")
        
        return self
    
    def predict_proba(self, X):
        """
        Predict probability estimates
        
        Parameters:
        -----------
        X : array-like, shape (m, n)
            Test features
            
        Returns:
        --------
        probabilities : array, shape (m,)
            Predicted probabilities
        """
        z = np.dot(X, self.weights) + self.bias
        return self._sigmoid(z)
    
    def predict(self, X, threshold=0.5):
        """
        Predict class labels
        
        Parameters:
        -----------
        X :  array-like, shape (m, n)
            Test features
        threshold : float, default=0.5
            Classification threshold
            
        Returns: 
        --------
        predictions : array, shape (m,)
            Predicted class labels (0 or 1)
        """
        probabilities = self. predict_proba(X)
        return (probabilities >= threshold).astype(int)
    
    def score(self, X, y):
        """
        Calculate accuracy score
        
        Parameters:
        -----------
        X : array-like, shape (m, n)
            Test features
        y : array-like, shape (m,)
            True labels
            
        Returns:
        --------
        accuracy : float
            Classification accuracy
        """
        predictions = self.predict(X)
        return np.mean(predictions == y)
    
    def plot_cost_history(self):
        """Plot cost vs iterations"""
        plt.figure(figsize=(10, 6))
        plt.plot(self.cost_history, linewidth=2)
        plt.xlabel('Iterations', fontsize=12)
        plt.ylabel('Cost', fontsize=12)
        plt.title('Cost Function over Iterations', fontsize=14, fontweight='bold')
        plt.grid(True, alpha=0.3)
        plt.show()

---

🔍 Method Breakdown

1️⃣ Initialization (__init__)

def __init__(self, learning_rate=0.01, n_iterations=1000, verbose=False):
    self.learning_rate = learning_rate      # α:  step size
    self.n_iterations = n_iterations        # Number of epochs
    self.verbose = verbose                  # Print progress
    self.weights = None                     # θ: feature weights
    self.bias = None                        # θ₀: intercept
    self.cost_history = []                  # Track learning

Purpose: Set up hyperparameters and initialize storage

---

2️⃣ Sigmoid Function (_sigmoid)

def _sigmoid(self, z):
    """
    Compute sigmoid:  σ(z) = 1 / (1 + e^(-z))
    
    Note: Clip z to prevent overflow
    """
    return 1 / (1 + np.exp(-np.clip(z, -500, 500)))

Key Points:

• ✅ Clipping prevents numerical overflow
• ✅ Vectorized for efficiency
• ✅ Maps (-∞, ∞) → (0, 1)

---

3️⃣ Cost Function (_compute_cost)

def _compute_cost(self, y_true, y_pred):
    """
    Binary cross-entropy:  J = -1/m Σ[y log(ŷ) + (1-y)log(1-ŷ)]
    """
    m = len(y_true)
    epsilon = 1e-15  # Prevent log(0)
    y_pred = np.clip(y_pred, epsilon, 1 - epsilon)
    
    cost = -1/m * np. sum(
        y_true * np.log(y_pred) + 
        (1 - y_true) * np.log(1 - y_pred)
    )
    return cost

Key Points:

• ✅ Epsilon prevents log(0) errors
• ✅ Lower cost = better predictions
• ✅ Convex function (single global minimum)

---

4️⃣ Training (fit)

def fit(self, X, y):
    """Main training loop"""
    m, n = X.shape
    
    # Initialize parameters to zero
    self.weights = np.zeros(n)
    self.bias = 0
    
    for i in range(self.n_iterations):
        # 1. Forward pass
        z = np.dot(X, self.weights) + self.bias
        predictions = self._sigmoid(z)
        
        # 2. Compute cost
        cost = self._compute_cost(y, predictions)
        self.cost_history. append(cost)
        
        # 3. Backward pass (gradients)
        dz = predictions - y
        dw = (1/m) * np.dot(X.T, dz)
        db = (1/m) * np.sum(dz)
        
        # 4. Update parameters
        self. weights -= self.learning_rate * dw
        self.bias -= self.learning_rate * db

Training Flow:

┌─────────────────────────────────────────┐
│ 1. Forward Pass: z = Xθ + b            │
│    ↓                                    │
│ 2. Activation: ŷ = σ(z)                │
│    ↓                                    │
│ 3.  Compute Cost: J(θ)                  │
│    ↓                                    │
│ 4. Gradients: ∇J = 1/m Xᵀ(ŷ - y)      │
│    ↓                                    │
│ 5. Update: θ := θ - α∇J                │
│    ↓                                    │
│ 6. Repeat...                           │
└─────────────────────────────────────────┘

---

5️⃣ Prediction (predict & predict_proba)

def predict_proba(self, X):
    """Return probabilities"""
    z = np.dot(X, self.weights) + self.bias
    return self._sigmoid(z)

def predict(self, X, threshold=0.5):
    """Return binary predictions"""
    probabilities = self.predict_proba(X)
    return (probabilities >= threshold).astype(int)

Decision Rule:

if P(y=1|x) ≥ 0.5:
    predict class 1
else:
    predict class 0

---

🎯 Usage Example

Complete Workflow

# Import libraries
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# 1. Load data
data = load_breast_cancer()
X, y = data.data, data. target

# 2. Split data
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 3. Scale features (important!)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# 4. Create and train model
model = LogisticRegression(
    learning_rate=0.01,
    n_iterations=1000,
    verbose=True
)
model.fit(X_train, y_train)

# 5. Make predictions
y_pred = model. predict(X_test)
probabilities = model.predict_proba(X_test)

# 6. Evaluate
accuracy = model.score(X_test, y_test)
print(f"\nTest Accuracy: {accuracy:.2%}")

# 7. Plot learning curve
model.plot_cost_history()

---

📊 Vectorization Benefits

Non-Vectorized (Slow ❌)

# Loop through each example - SLOW! 
for i in range(m):
    z[i] = 0
    for j in range(n):
        z[i] += X[i][j] * weights[j]
    z[i] += bias

Vectorized (Fast ✅)

# Single line - FAST!
z = np. dot(X, weights) + bias

Speed Comparison:

Dataset Size	Loop Time	Vectorized Time	Speedup
1,000 samples	100ms	1ms	100x faster
10,000 samples	1000ms	5ms	200x faster
100,000 samples	10s	50ms	200x faster

---

🔧 Hyperparameter Tuning

Learning Rate

# Try different learning rates
learning_rates = [0.001, 0.01, 0.1, 1.0]

for lr in learning_rates: 
    model = LogisticRegression(learning_rate=lr, n_iterations=1000)
    model.fit(X_train, y_train)
    accuracy = model. score(X_test, y_test)
    print(f"LR={lr}:  Accuracy={accuracy:.2%}")

Number of Iterations

# Try different iteration counts
iterations = [100, 500, 1000, 2000]

for iters in iterations:
    model = LogisticRegression(learning_rate=0.01, n_iterations=iters)
    model.fit(X_train, y_train)
    accuracy = model.score(X_test, y_test)
    print(f"Iterations={iters}:  Accuracy={accuracy:.2%}")

---

🐛 Common Pitfalls

1. Not Scaling Features

# ❌ BAD:  Different feature scales
X_train = [[1, 1000], [2, 2000]]  # Feature 2 dominates! 

# ✅ GOOD: Standardized features
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)

2. Learning Rate Too High

# ❌ BAD: Cost oscillates
model = LogisticRegression(learning_rate=10.0)

# ✅ GOOD: Stable convergence
model = LogisticRegression(learning_rate=0.01)

3. Not Enough Iterations

# ❌ BAD: Model doesn't converge
model = LogisticRegression(n_iterations=10)

# ✅ GOOD: Sufficient training
model = LogisticRegression(n_iterations=1000)

---

💡 Best Practices

✅ Do Scale/normalize features Use appropriate learning rate Monitor cost function Split data properly Use vectorized operations Set random seed for reproducibility

❌ Don't Train on unscaled data Use extreme learning rates Ignore convergence issues Test on training data Use loops instead of vectorization Forget to validate results

---

🧪 Testing Your Implementation

# Unit tests
def test_sigmoid():
    assert model._sigmoid(0) == 0.5
    assert model._sigmoid(1000) < 1. 0
    assert model._sigmoid(-1000) > 0.0

def test_predictions():
    model = LogisticRegression()
    X = np.array([[1, 2], [3, 4]])
    y = np.array([0, 1])
    model.fit(X, y)
    preds = model.predict(X)
    assert len(preds) == len(y)
    assert all(p in [0, 1] for p in preds)

# Run tests
test_sigmoid()
test_predictions()
print("✅ All tests passed!")

---

🔗 Next Steps

Ready to see beautiful visualizations?

← Mathematical Foundation | Visualizations →