fixed testing (#11)

Author: jamilgafur

Adversarial_Observation/Attacks.py

@@ -3,7 +3,7 @@
 import os
 from datetime import datetime
 from torch.nn import Softmax
-from .utils import fgsm_attack, pgd_attack, compute_success_rate, log_metrics, visualize_adversarial_examples
+from .utils import fgsm_attack, compute_success_rate, log_metrics, visualize_adversarial_examples
 from .utils import seed_everything
 
 class AdversarialTester:
@@ -44,8 +44,6 @@ def _generate_adversarial_images(self, input_batch_data: torch.Tensor):
         logging.info(f"Starting attack with method: {self.attack_method}")
         if self.attack_method == 'fgsm':
             return fgsm_attack(input_batch_data, self.model, self.epsilon, self.device)
-        elif self.attack_method == 'pgd':
-            return pgd_attack(input_batch_data, self.model, self.epsilon, self.alpha, self.num_steps, self.device)
         else:
             raise ValueError(f"Unsupported attack method: {self.attack_method}")
 

Adversarial_Observation/utils.py

@@ -60,51 +60,6 @@ def fgsm_attack(input_data: torch.Tensor, model: torch.nn.Module, epsilon: float
 
     return adversarial_data
 
-# PGD Attack (Projected Gradient Descent)
-def pgd_attack(input_data: torch.Tensor, model: torch.nn.Module, epsilon: float, alpha: float, num_steps: int, device: torch.device) -> torch.Tensor:
-    """
-    Performs PGD attack on the input data.
-    
-    Args:
-        input_data (torch.Tensor): The original input batch.
-        model (torch.nn.Module): The model to attack.
-        epsilon (float): The maximum perturbation magnitude.
-        alpha (float): The step size for each iteration.
-        num_steps (int): The number of steps for the attack.
-        device (torch.device): The device to perform the attack on (cuda or cpu).
-        
-    Returns:
-        torch.Tensor: The adversarially perturbed batch.
-    """
-    # Initialize the perturbation to be the same as the original input data
-    perturbed_data = input_data.clone().detach()
-    perturbed_data.requires_grad = True
-
-    for _ in range(num_steps):
-        # Forward pass to get the model's output
-        output = model(perturbed_data)
-        
-        # Compute the loss (we assume classification, so we use cross-entropy loss)
-        loss = F.cross_entropy(output, torch.argmax(output, dim=1))
-
-        # Backpropagate the gradients
-        model.zero_grad()
-        loss.backward()
-
-        # Get the gradient of the input data with respect to the loss
-        gradient = perturbed_data.grad.data
-
-        # Update the perturbation (step in the direction of the gradient)
-        perturbed_data = perturbed_data + alpha * torch.sign(gradient)
-        
-        # Clip the perturbation to stay within the epsilon ball
-        perturbed_data = torch.max(torch.min(perturbed_data, input_data + epsilon), input_data - epsilon)
-
-        # Ensure the adversarial data is within the valid range (e.g., [0, 1] for image data)
-        perturbed_data = torch.clamp(perturbed_data, 0, 1)
-
-    return perturbed_data
-
 # Compute the success rate of the attack
 def compute_success_rate(original_preds: torch.Tensor, adversarial_preds: torch.Tensor) -> float:
     """

README.md

@@ -104,7 +104,7 @@ The attack will run for `num_iterations` iterations, and the results will be sav
 **Example:**
 
 ```bash
-python main.py --iterations 50 --particles 100 --save_dir "analysis_results"
+python taint_MNIST.py --iterations 50 --particles 100 --save_dir "analysis_results"
 ```
 
 This command performs the attack with **50 iterations** and **100 particles**.
@@ -151,6 +151,22 @@ After the attack is complete, the following information is saved:
 
 You can open the `attack_analysis.json` file for a detailed analysis of the attack.
 
+---
+## Citing This Work
+
+If you use or refer to this code in your research, please cite the following paper:
+
+
+```
+@incollection{gafur2024adversarial,
+  title={Adversarial Robustness and Explainability of Machine Learning Models},
+  author={Gafur, Jamil and Goddard, Steve and Lai, William},
+  booktitle={Practice and Experience in Advanced Research Computing 2024: Human Powered Computing},
+  pages={1--7},
+  year={2024}
+}
+```
+
 ---
 
 ## Contributing

requirements.txt

@@ -12,3 +12,5 @@ shap
 torch
 torchvision
 sphinx_rtd_theme
+tensorflow
+imageio[pyav]

setup.py

@@ -19,7 +19,6 @@
     license='MIT',
     packages=[  
                 "Adversarial_Observation", 
-                "Swarm_Observer"
             ],
     python_requires='>=3.6',
     install_requires=[

tests/__init__.py

@@ -0,0 +1,36 @@
+# tests/test_adversarial_attacks.py
+
+import torch
+import pytest
+from Adversarial_Observation.utils import fgsm_attack, load_MNIST_model
+from torch import nn
+
+# Helper function to generate random image data and a simple model
+def get_sample_data():
+    model = load_MNIST_model()
+    # Fake image (1, 1, 28, 28) for MNIST (batch_size=1, channel=1, height=28, width=28)
+    input_data = torch.rand((1, 1, 28, 28))  # Random image
+    return input_data, model
+
+def test_fgsm_attack():
+    input_data, model = get_sample_data()
+    epsilon = 0.1  # Perturbation size
+    device = torch.device('cpu')  # Using CPU for simplicity
+
+    # Apply FGSM attack
+    adversarial_data = fgsm_attack(input_data, model, epsilon, device)
+
+    # Check if adversarial data has been perturbed (should not be equal to original data)
+    assert not torch.allclose(input_data, adversarial_data, atol=1e-5), "FGSM attack failed to perturb the input"
+    
+def test_success_rate():
+    # Testing the success rate calculation
+    original_preds = torch.tensor([0, 1, 2])  # Some dummy predictions
+    adversarial_preds = torch.tensor([1, 0, 2])  # Adversarial predictions
+
+    # Calculate success rate (should be 2/3 as two predictions are different)
+    from Adversarial_Observation.utils import compute_success_rate
+    success_rate = compute_success_rate(original_preds, adversarial_preds)
+    
+    assert success_rate == 2/3, f"Expected success rate 2/3, but got {success_rate}"
+