Add standard deviation to PDTP output

Signed-off-by: abigailt <[email protected]>

Author: abigailgold

art/metrics/privacy/membership_leakage.py

@@ -37,7 +37,7 @@ def PDTP(  # pylint: disable=C0103
     y: np.ndarray,
     indexes: Optional[np.ndarray] = None,
     num_iter: Optional[int] = 10,
-) -> Tuple[np.ndarray, np.ndarray]:
+) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
     """
     Compute the pointwise differential training privacy metric for the given classifier and training set.
 
@@ -52,8 +52,8 @@ def PDTP(  # pylint: disable=C0103
                     computed for all samples in `x`.
     :param num_iter: the number of iterations of PDTP computation to run for each sample. If not supplied,
                      defaults to 10. The result is the average across iterations.
-    :return: A tuple of two arrays, containing the average (worse) PDTP value for each sample in the training set
-             respectively. The higher the value, the higher the privacy leakage for that sample.
+    :return: A tuple of three arrays, containing the average (worse, standard deviation) PDTP value for each sample in
+             the training set respectively. The higher the value, the higher the privacy leakage for that sample.
     """
     from art.estimators.classification.pytorch import PyTorchClassifier
     from art.estimators.classification.tensorflow import TensorFlowV2Classifier
@@ -121,6 +121,7 @@ def PDTP(  # pylint: disable=C0103
     per_sample = list(map(list, zip(*results)))
     avg_per_sample = np.array([sum(val) / len(val) for val in per_sample])
     worse_per_sample = np.max(per_sample, axis=1)
+    std_dev_per_sample = np.std(per_sample, axis=1)
 
-    # return avg+worse leakage per sample
-    return avg_per_sample, worse_per_sample
+    # return avg+worse leakage + standard deviation per sample
+    return avg_per_sample, worse_per_sample, std_dev_per_sample

tests/metrics/privacy/test_membership_leakage.py

@@ -35,14 +35,15 @@ def test_membership_leakage_decision_tree(art_warning, decision_tree_estimator,
         extra_classifier = decision_tree_estimator()
         (x_train, y_train), _ = get_iris_dataset
         prev = classifier.model.tree_
-        avg_leakage, worse_leakage = PDTP(classifier, extra_classifier, x_train, y_train)
+        avg_leakage, worse_leakage, std_dev = PDTP(classifier, extra_classifier, x_train, y_train)
         logger.info("Average PDTP leakage: %.2f", (np.average(avg_leakage)))
         logger.info("Max PDTP leakage: %.2f", (np.max(avg_leakage)))
         assert classifier.model.tree_ == prev
         assert np.all(avg_leakage >= 1.0)
         assert np.all(worse_leakage >= avg_leakage)
         assert avg_leakage.shape[0] == x_train.shape[0]
         assert worse_leakage.shape[0] == x_train.shape[0]
+        assert std_dev.shape[0] == x_train.shape[0]
     except ARTTestException as e:
         art_warning(e)
 
@@ -53,13 +54,14 @@ def test_membership_leakage_tabular(art_warning, tabular_dl_estimator, get_iris_
         classifier = tabular_dl_estimator()
         extra_classifier = tabular_dl_estimator()
         (x_train, y_train), _ = get_iris_dataset
-        avg_leakage, worse_leakage = PDTP(classifier, extra_classifier, x_train, y_train)
+        avg_leakage, worse_leakage, std_dev = PDTP(classifier, extra_classifier, x_train, y_train)
         logger.info("Average PDTP leakage: %.2f", (np.average(avg_leakage)))
         logger.info("Max PDTP leakage: %.2f", (np.max(avg_leakage)))
         assert np.all(avg_leakage >= 1.0)
         assert np.all(worse_leakage >= avg_leakage)
         assert avg_leakage.shape[0] == x_train.shape[0]
         assert worse_leakage.shape[0] == x_train.shape[0]
+        assert std_dev.shape[0] == x_train.shape[0]
     except ARTTestException as e:
         art_warning(e)
 
@@ -71,13 +73,16 @@ def test_membership_leakage_image(art_warning, image_dl_estimator, get_default_m
         extra_classifier, _ = image_dl_estimator()
         (x_train, y_train), _ = get_default_mnist_subset
         indexes = random.sample(range(x_train.shape[0]), 100)
-        avg_leakage, worse_leakage = PDTP(classifier, extra_classifier, x_train, y_train, indexes=indexes, num_iter=1)
+        avg_leakage, worse_leakage, std_dev = PDTP(
+            classifier, extra_classifier, x_train, y_train, indexes=indexes, num_iter=1
+        )
         logger.info("Average PDTP leakage: %.2f", (np.average(avg_leakage)))
         logger.info("Max PDTP leakage: %.2f", (np.max(avg_leakage)))
         assert np.all(avg_leakage >= 1.0)
         assert np.all(worse_leakage >= avg_leakage)
         assert avg_leakage.shape[0] == x_train.shape[0]
         assert worse_leakage.shape[0] == x_train.shape[0]
+        assert std_dev.shape[0] == x_train.shape[0]
     except ARTTestException as e:
         art_warning(e)